apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  45 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  46
  47 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  48                             MD5SumValue const &CurMd5, std::string const &CurLang);
  49
  50 using std::string;
  51
  52 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We set the dirty flag and make sure that is written to the disk */
  55 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  56                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  57                     FoundFileDeps(0)
  58 {
  59    CurrentFile = 0;
  60    memset(UniqHash,0,sizeof(UniqHash));
  61
  62    if (_error->PendingError() == true)
  63       return;
  64
  65    if (Map.Size() == 0)
  66    {
  67       // Setup the map interface..
  68       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  69       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  77       Cache.HeaderP->VerSysName = idxVerSysName;
  78       // this pointer is set in ReMap, but we need it now for WriteUniqString
  79       Cache.StringItemP = (pkgCache::StringItem *)Map.Data();
  80       map_ptrloc const idxArchitecture = WriteUniqString(_config->Find("APT::Architecture"));
  81       Cache.HeaderP->Architecture = idxArchitecture;
  82       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  83          return;
  84       Cache.ReMap();
  85    }
  86    else
  87    {
  88       // Map directly from the existing file
  89       Cache.ReMap();
  90       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  91       if (Cache.VS != _system->VS)
  92       {
  93          _error->Error(_("Cache has an incompatible versioning system"));
  94          return;
  95       }
  96    }
  97
  98    Cache.HeaderP->Dirty = true;
  99    Map.Sync(0,sizeof(pkgCache::Header));
 100 }
 101                                                                         /*}}}*/
 102 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 103 // ---------------------------------------------------------------------
 104 /* We sync the data then unset the dirty flag in two steps so as to
 105    advoid a problem during a crash */
 106 pkgCacheGenerator::~pkgCacheGenerator()
 107 {
 108    if (_error->PendingError() == true)
 109       return;
 110    if (Map.Sync() == false)
 111       return;
 112
 113    Cache.HeaderP->Dirty = false;
 114    Cache.HeaderP->CacheFileSize = Map.Size();
 115    Map.Sync(0,sizeof(pkgCache::Header));
 116 }
 117                                                                         /*}}}*/
 118 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 119    if (oldMap == newMap)
 120       return;
 121
 122    if (_config->FindB("Debug::pkgCacheGen", false))
 123       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 124
 125    Cache.ReMap(false);
 126
 127    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 128
 129    for (size_t i = 0; i < _count(UniqHash); ++i)
 130       if (UniqHash[i] != 0)
 131          UniqHash[i] += (pkgCache::StringItem const * const) newMap - (pkgCache::StringItem const * const) oldMap;
 132
 133    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 134         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 135       (*i)->ReMap(oldMap, newMap);
 136    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 137         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 138       (*i)->ReMap(oldMap, newMap);
 139    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 140         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 141       (*i)->ReMap(oldMap, newMap);
 142    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 143         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 144       (*i)->ReMap(oldMap, newMap);
 145    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 146         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 147       (*i)->ReMap(oldMap, newMap);
 148    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 149         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 150       (*i)->ReMap(oldMap, newMap);
 151    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 152         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 153       (*i)->ReMap(oldMap, newMap);
 154 }                                                                       /*}}}*/
 155 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 156 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 157                                         const unsigned long &Len) {
 158    void const * const oldMap = Map.Data();
 159    map_ptrloc const index = Map.WriteString(String, Len);
 160    if (index != 0)
 161       ReMap(oldMap, Map.Data());
 162    return index;
 163 }
 164                                                                         /*}}}*/
 165 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 166 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 167    void const * const oldMap = Map.Data();
 168    map_ptrloc const index = Map.WriteString(String);
 169    if (index != 0)
 170       ReMap(oldMap, Map.Data());
 171    return index;
 172 }
 173                                                                         /*}}}*/
 174 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 175    void const * const oldMap = Map.Data();
 176    map_ptrloc const index = Map.Allocate(size);
 177    if (index != 0)
 178       ReMap(oldMap, Map.Data());
 179    return index;
 180 }
 181                                                                         /*}}}*/
 182 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 183 // ---------------------------------------------------------------------
 184 /* This provides the generation of the entries in the cache. Each loop
 185    goes through a single package record from the underlying parse engine. */
 186 bool pkgCacheGenerator::MergeList(ListParser &List,
 187                                   pkgCache::VerIterator *OutVer)
 188 {
 189    List.Owner = this;
 190
 191    unsigned int Counter = 0;
 192    while (List.Step() == true)
 193    {
 194       string const PackageName = List.Package();
 195       if (PackageName.empty() == true)
 196          return false;
 197
 198       Counter++;
 199       if (Counter % 100 == 0 && Progress != 0)
 200          Progress->Progress(List.Offset());
 201
 202       string Arch = List.Architecture();
 203       string const Version = List.Version();
 204       if (Version.empty() == true && Arch.empty() == true)
 205       {
 206          // package descriptions
 207          if (MergeListGroup(List, PackageName) == false)
 208             return false;
 209          continue;
 210       }
 211
 212       if (Arch.empty() == true)
 213       {
 214          // use the pseudo arch 'none' for arch-less packages
 215          Arch = "none";
 216          /* We might built a SingleArchCache here, which we don't want to blow up
 217             just for these :none packages to a proper MultiArchCache, so just ensure
 218             that we have always a native package structure first for SingleArch */
 219          pkgCache::PkgIterator NP;
 220          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 221          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 222          // TRANSLATOR: The first placeholder is a package name,
 223          // the other two should be copied verbatim as they include debug info
 224          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 225                               PackageName.c_str(), "NewPackage", 0);
 226       }
 227
 228       // Get a pointer to the package structure
 229       pkgCache::PkgIterator Pkg;
 230       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 231       if (NewPackage(Pkg, PackageName, Arch) == false)
 232          // TRANSLATOR: The first placeholder is a package name,
 233          // the other two should be copied verbatim as they include debug info
 234          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 235                               PackageName.c_str(), "NewPackage", 1);
 236
 237
 238       if (Version.empty() == true)
 239       {
 240          if (MergeListPackage(List, Pkg) == false)
 241             return false;
 242       }
 243       else
 244       {
 245          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 246             return false;
 247       }
 248
 249       if (OutVer != 0)
 250       {
 251          FoundFileDeps |= List.HasFileDeps();
 252          return true;
 253       }
 254    }
 255
 256    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 257       return _error->Error(_("Wow, you exceeded the number of package "
 258                              "names this APT is capable of."));
 259    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 260       return _error->Error(_("Wow, you exceeded the number of versions "
 261                              "this APT is capable of."));
 262    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 263       return _error->Error(_("Wow, you exceeded the number of descriptions "
 264                              "this APT is capable of."));
 265    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 266       return _error->Error(_("Wow, you exceeded the number of dependencies "
 267                              "this APT is capable of."));
 268
 269    FoundFileDeps |= List.HasFileDeps();
 270    return true;
 271 }
 272 // CacheGenerator::MergeListGroup                                       /*{{{*/
 273 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 274 {
 275    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 276    // a group has no data on it's own, only packages have it but these
 277    // stanzas like this come from Translation- files to add descriptions,
 278    // but without a version we don't need a description for it…
 279    if (Grp.end() == true)
 280       return true;
 281    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 282
 283    pkgCache::PkgIterator Pkg;
 284    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 285    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 286       if (MergeListPackage(List, Pkg) == false)
 287          return false;
 288
 289    return true;
 290 }
 291                                                                         /*}}}*/
 292 // CacheGenerator::MergeListPackage                                     /*{{{*/
 293 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 294 {
 295    // we first process the package, then the descriptions
 296    // (for deb this package processing is in fact a no-op)
 297    pkgCache::VerIterator Ver(Cache);
 298    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 299    if (List.UsePackage(Pkg, Ver) == false)
 300       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 301                            Pkg.Name(), "UsePackage", 1);
 302
 303    // Find the right version to write the description
 304    MD5SumValue CurMd5 = List.Description_md5();
 305    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 306       return true;
 307    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 308    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 309    {
 310       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 311
 312       // a version can only have one md5 describing it
 313       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 314          continue;
 315
 316       map_ptrloc md5idx = VerDesc->md5sum;
 317       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 318       {
 319          // don't add a new description if we have one for the given
 320          // md5 && language
 321          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 322             continue;
 323
 324          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 325       }
 326
 327       // we can stop here as all "same" versions will share the description
 328       break;
 329    }
 330
 331    return true;
 332 }
 333                                                                         /*}}}*/
 334 // CacheGenerator::MergeListVersion                                     /*{{{*/
 335 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 336                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 337 {
 338    pkgCache::VerIterator Ver = Pkg.VersionList();
 339    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 340    map_ptrloc *LastVer = &Pkg->VersionList;
 341    void const * oldMap = Map.Data();
 342
 343    unsigned short const Hash = List.VersionHash();
 344    if (Ver.end() == false)
 345    {
 346       /* We know the list is sorted so we use that fact in the search.
 347          Insertion of new versions is done with correct sorting */
 348       int Res = 1;
 349       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 350       {
 351          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 352          // Version is higher as current version - insert here
 353          if (Res > 0)
 354             break;
 355          // Versionstrings are equal - is hash also equal?
 356          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 357             break;
 358          // proceed with the next till we have either the right
 359          // or we found another version (which will be lower)
 360       }
 361
 362       /* We already have a version for this item, record that we saw it */
 363       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 364       {
 365          if (List.UsePackage(Pkg,Ver) == false)
 366             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 367                                  Pkg.Name(), "UsePackage", 2);
 368
 369          if (NewFileVer(Ver,List) == false)
 370             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 371                                  Pkg.Name(), "NewFileVer", 1);
 372
 373          // Read only a single record and return
 374          if (OutVer != 0)
 375          {
 376             *OutVer = Ver;
 377             return true;
 378          }
 379
 380          return true;
 381       }
 382    }
 383
 384    // Add a new version
 385    map_ptrloc const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 386    if (verindex == 0 && _error->PendingError())
 387       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 388                            Pkg.Name(), "NewVersion", 1);
 389
 390    if (oldMap != Map.Data())
 391          LastVer += (map_ptrloc const * const) Map.Data() - (map_ptrloc const * const) oldMap;
 392    *LastVer = verindex;
 393
 394    if (unlikely(List.NewVersion(Ver) == false))
 395       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 396                            Pkg.Name(), "NewVersion", 2);
 397
 398    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 399       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 400                            Pkg.Name(), "UsePackage", 3);
 401
 402    if (unlikely(NewFileVer(Ver,List) == false))
 403       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 404                            Pkg.Name(), "NewFileVer", 2);
 405
 406    pkgCache::GrpIterator Grp = Pkg.Group();
 407    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 408
 409    /* If it is the first version of this package we need to add implicit
 410       Multi-Arch dependencies to all other package versions in the group now -
 411       otherwise we just add them for this new version */
 412    if (Pkg.VersionList()->NextVer == 0)
 413    {
 414       pkgCache::PkgIterator P = Grp.PackageList();
 415       Dynamic<pkgCache::PkgIterator> DynP(P);
 416       for (; P.end() != true; P = Grp.NextPkg(P))
 417       {
 418          if (P->ID == Pkg->ID)
 419             continue;
 420          pkgCache::VerIterator V = P.VersionList();
 421          Dynamic<pkgCache::VerIterator> DynV(V);
 422          for (; V.end() != true; ++V)
 423             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 424                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 425                                     Pkg.Name(), "AddImplicitDepends", 1);
 426       }
 427       /* :none packages are packages without an architecture. They are forbidden by
 428          debian-policy, so usually they will only be in (old) dpkg status files -
 429          and dpkg will complain about them - and are pretty rare. We therefore do
 430          usually not create conflicts while the parent is created, but only if a :none
 431          package (= the target) appears. This creates incorrect dependencies on :none
 432          for architecture-specific dependencies on the package we copy from, but we
 433          will ignore this bug as architecture-specific dependencies are only allowed
 434          in jessie and until then the :none packages should be extinct (hopefully).
 435          In other words: This should work long enough to allow graceful removal of
 436          these packages, it is not supposed to allow users to keep using them … */
 437       if (strcmp(Pkg.Arch(), "none") == 0)
 438       {
 439          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 440          if (M.end() == false && Pkg != M)
 441          {
 442             pkgCache::DepIterator D = M.RevDependsList();
 443             Dynamic<pkgCache::DepIterator> DynD(D);
 444             for (; D.end() == false; ++D)
 445             {
 446                if ((D->Type != pkgCache::Dep::Conflicts &&
 447                     D->Type != pkgCache::Dep::DpkgBreaks &&
 448                     D->Type != pkgCache::Dep::Replaces) ||
 449                    D.ParentPkg().Group() == Grp)
 450                   continue;
 451
 452                map_ptrloc *OldDepLast = NULL;
 453                pkgCache::VerIterator ConVersion = D.ParentVer();
 454                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 455                // duplicate the Conflicts/Breaks/Replaces for :none arch
 456                NewDepends(Pkg, ConVersion, D->Version,
 457                      D->CompareOp, D->Type, OldDepLast);
 458             }
 459          }
 460       }
 461    }
 462    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 463       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 464                            Pkg.Name(), "AddImplicitDepends", 2);
 465
 466    // Read only a single record and return
 467    if (OutVer != 0)
 468    {
 469       *OutVer = Ver;
 470       return true;
 471    }
 472
 473    /* Record the Description(s) based on their master md5sum */
 474    MD5SumValue CurMd5 = List.Description_md5();
 475    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 476       return true;
 477
 478    /* Before we add a new description we first search in the group for
 479       a version with a description of the same MD5 - if so we reuse this
 480       description group instead of creating our own for this version */
 481    for (pkgCache::PkgIterator P = Grp.PackageList();
 482         P.end() == false; P = Grp.NextPkg(P))
 483    {
 484       for (pkgCache::VerIterator V = P.VersionList();
 485            V.end() == false; ++V)
 486       {
 487          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 488             continue;
 489          Ver->DescriptionList = V->DescriptionList;
 490       }
 491    }
 492
 493    // We haven't found reusable descriptions, so add the first description(s)
 494    map_ptrloc md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 495    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 496    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 497       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 498          return false;
 499    return true;
 500 }
 501                                                                         /*}}}*/
 502 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_ptrloc &md5idx) /*{{{*/
 503 {
 504    pkgCache::DescIterator Desc;
 505    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 506
 507    map_ptrloc const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 508    if (unlikely(descindex == 0 && _error->PendingError()))
 509       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 510             Ver.ParentPkg().Name(), "NewDescription", 1);
 511
 512    md5idx = Desc->md5sum;
 513    Desc->ParentPkg = Ver.ParentPkg().Index();
 514
 515    // we add at the end, so that the start is constant as we need
 516    // that to be able to efficiently share these lists
 517    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 518    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 519    map_ptrloc * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 520    *LastNextDesc = descindex;
 521
 522    if (NewFileDesc(Desc,List) == false)
 523       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 524             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 525
 526    return true;
 527 }
 528                                                                         /*}}}*/
 529                                                                         /*}}}*/
 530 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 531 // ---------------------------------------------------------------------
 532 /* If we found any file depends while parsing the main list we need to
 533    resolve them. Since it is undesired to load the entire list of files
 534    into the cache as virtual packages we do a two stage effort. MergeList
 535    identifies the file depends and this creates Provdies for them by
 536    re-parsing all the indexs. */
 537 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 538 {
 539    List.Owner = this;
 540
 541    unsigned int Counter = 0;
 542    while (List.Step() == true)
 543    {
 544       string PackageName = List.Package();
 545       if (PackageName.empty() == true)
 546          return false;
 547       string Version = List.Version();
 548       if (Version.empty() == true)
 549          continue;
 550
 551       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 552       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 553       if (Pkg.end() == true)
 554          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 555                                 PackageName.c_str(), "FindPkg", 1);
 556       Counter++;
 557       if (Counter % 100 == 0 && Progress != 0)
 558          Progress->Progress(List.Offset());
 559
 560       unsigned short Hash = List.VersionHash();
 561       pkgCache::VerIterator Ver = Pkg.VersionList();
 562       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 563       for (; Ver.end() == false; ++Ver)
 564       {
 565          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 566          {
 567             if (List.CollectFileProvides(Cache,Ver) == false)
 568                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 569                                     PackageName.c_str(), "CollectFileProvides", 1);
 570             break;
 571          }
 572       }
 573
 574       if (Ver.end() == true)
 575          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 576    }
 577
 578    return true;
 579 }
 580                                                                         /*}}}*/
 581 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* This creates a new group structure and adds it to the hash table */
 584 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 585 {
 586    Grp = Cache.FindGrp(Name);
 587    if (Grp.end() == false)
 588       return true;
 589
 590    // Get a structure
 591    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 592    if (unlikely(Group == 0))
 593       return false;
 594
 595    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 596    map_ptrloc const idxName = WriteStringInMap(Name);
 597    if (unlikely(idxName == 0))
 598       return false;
 599    Grp->Name = idxName;
 600
 601    // Insert it into the hash table
 602    unsigned long const Hash = Cache.Hash(Name);
 603    map_ptrloc *insertAt = &Cache.HeaderP->GrpHashTable[Hash];
 604    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 605       insertAt = &(Cache.GrpP + *insertAt)->Next;
 606    Grp->Next = *insertAt;
 607    *insertAt = Group;
 608
 609    Grp->ID = Cache.HeaderP->GroupCount++;
 610    return true;
 611 }
 612                                                                         /*}}}*/
 613 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 614 // ---------------------------------------------------------------------
 615 /* This creates a new package structure and adds it to the hash table */
 616 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 617                                         const string &Arch) {
 618    pkgCache::GrpIterator Grp;
 619    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 620    if (unlikely(NewGroup(Grp, Name) == false))
 621       return false;
 622
 623    Pkg = Grp.FindPkg(Arch);
 624       if (Pkg.end() == false)
 625          return true;
 626
 627    // Get a structure
 628    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 629    if (unlikely(Package == 0))
 630       return false;
 631    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 632
 633    // Insert the package into our package list
 634    if (Grp->FirstPackage == 0) // the group is new
 635    {
 636       Grp->FirstPackage = Package;
 637       // Insert it into the hash table
 638       unsigned long const Hash = Cache.Hash(Name);
 639       map_ptrloc *insertAt = &Cache.HeaderP->PkgHashTable[Hash];
 640       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.PkgP + *insertAt)->Name) > 0)
 641          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 642       Pkg->NextPackage = *insertAt;
 643       *insertAt = Package;
 644    }
 645    else // Group the Packages together
 646    {
 647       // this package is the new last package
 648       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 649       Pkg->NextPackage = LastPkg->NextPackage;
 650       LastPkg->NextPackage = Package;
 651    }
 652    Grp->LastPackage = Package;
 653
 654    // Set the name, arch and the ID
 655    Pkg->Name = Grp->Name;
 656    Pkg->Group = Grp.Index();
 657    // all is mapped to the native architecture
 658    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 659    if (unlikely(idxArch == 0))
 660       return false;
 661    Pkg->Arch = idxArch;
 662    Pkg->ID = Cache.HeaderP->PackageCount++;
 663
 664    return true;
 665 }
 666                                                                         /*}}}*/
 667 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 668 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 669                                            pkgCache::PkgIterator &P,
 670                                            pkgCache::VerIterator &V)
 671 {
 672    // copy P.Arch() into a string here as a cache remap
 673    // in NewDepends() later may alter the pointer location
 674    string Arch = P.Arch() == NULL ? "" : P.Arch();
 675    map_ptrloc *OldDepLast = NULL;
 676    /* MultiArch handling introduces a lot of implicit Dependencies:
 677       - MultiArch: same → Co-Installable if they have the same version
 678       - All others conflict with all other group members */
 679    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 680    pkgCache::PkgIterator D = G.PackageList();
 681    Dynamic<pkgCache::PkgIterator> DynD(D);
 682    map_ptrloc const VerStrIdx = V->VerStr;
 683    for (; D.end() != true; D = G.NextPkg(D))
 684    {
 685       if (Arch == D.Arch() || D->VersionList == 0)
 686          continue;
 687       /* We allow only one installed arch at the time
 688          per group, therefore each group member conflicts
 689          with all other group members */
 690       if (coInstall == true)
 691       {
 692          // Replaces: ${self}:other ( << ${binary:Version})
 693          NewDepends(D, V, VerStrIdx,
 694                     pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 695                     OldDepLast);
 696          // Breaks: ${self}:other (!= ${binary:Version})
 697          NewDepends(D, V, VerStrIdx,
 698                     pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 699                     OldDepLast);
 700       } else {
 701          // Conflicts: ${self}:other
 702          NewDepends(D, V, 0,
 703                     pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 704                     OldDepLast);
 705       }
 706    }
 707    return true;
 708 }
 709 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 710                                            pkgCache::PkgIterator &D)
 711 {
 712    /* MultiArch handling introduces a lot of implicit Dependencies:
 713       - MultiArch: same → Co-Installable if they have the same version
 714       - All others conflict with all other group members */
 715    map_ptrloc *OldDepLast = NULL;
 716    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 717    if (coInstall == true)
 718    {
 719       map_ptrloc const VerStrIdx = V->VerStr;
 720       // Replaces: ${self}:other ( << ${binary:Version})
 721       NewDepends(D, V, VerStrIdx,
 722                  pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 723                  OldDepLast);
 724       // Breaks: ${self}:other (!= ${binary:Version})
 725       NewDepends(D, V, VerStrIdx,
 726                  pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 727                  OldDepLast);
 728    } else {
 729       // Conflicts: ${self}:other
 730       NewDepends(D, V, 0,
 731                  pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 732                  OldDepLast);
 733    }
 734    return true;
 735 }
 736
 737                                                                         /*}}}*/
 738 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 739 // ---------------------------------------------------------------------
 740 /* */
 741 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 742                                    ListParser &List)
 743 {
 744    if (CurrentFile == 0)
 745       return true;
 746
 747    // Get a structure
 748    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 749    if (VerFile == 0)
 750       return 0;
 751
 752    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 753    VF->File = CurrentFile - Cache.PkgFileP;
 754
 755    // Link it to the end of the list
 756    map_ptrloc *Last = &Ver->FileList;
 757    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 758       Last = &V->NextFile;
 759    VF->NextFile = *Last;
 760    *Last = VF.Index();
 761
 762    VF->Offset = List.Offset();
 763    VF->Size = List.Size();
 764    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 765       Cache.HeaderP->MaxVerFileSize = VF->Size;
 766    Cache.HeaderP->VerFileCount++;
 767
 768    return true;
 769 }
 770                                                                         /*}}}*/
 771 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 772 // ---------------------------------------------------------------------
 773 /* This puts a version structure in the linked list */
 774 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 775                                             const string &VerStr,
 776                                             map_ptrloc const ParentPkg,
 777                                             unsigned long const Hash,
 778                                             unsigned long Next)
 779 {
 780    // Get a structure
 781    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 782    if (Version == 0)
 783       return 0;
 784
 785    // Fill it in
 786    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 787    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 788    Ver->NextVer = Next;
 789    Ver->ParentPkg = ParentPkg;
 790    Ver->Hash = Hash;
 791    Ver->ID = Cache.HeaderP->VersionCount++;
 792
 793    // try to find the version string in the group for reuse
 794    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 795    pkgCache::GrpIterator Grp = Pkg.Group();
 796    if (Pkg.end() == false && Grp.end() == false)
 797    {
 798       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 799       {
 800          if (Pkg == P)
 801             continue;
 802          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 803          {
 804             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 805             if (cmp == 0)
 806             {
 807                Ver->VerStr = V->VerStr;
 808                return Version;
 809             }
 810             else if (cmp < 0)
 811                break;
 812          }
 813       }
 814    }
 815    // haven't found the version string, so create
 816    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 817    if (unlikely(idxVerStr == 0))
 818       return 0;
 819    Ver->VerStr = idxVerStr;
 820    return Version;
 821 }
 822                                                                         /*}}}*/
 823 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 824 // ---------------------------------------------------------------------
 825 /* */
 826 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 827                                    ListParser &List)
 828 {
 829    if (CurrentFile == 0)
 830       return true;
 831
 832    // Get a structure
 833    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 834    if (DescFile == 0)
 835       return false;
 836
 837    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 838    DF->File = CurrentFile - Cache.PkgFileP;
 839
 840    // Link it to the end of the list
 841    map_ptrloc *Last = &Desc->FileList;
 842    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 843       Last = &D->NextFile;
 844
 845    DF->NextFile = *Last;
 846    *Last = DF.Index();
 847
 848    DF->Offset = List.Offset();
 849    DF->Size = List.Size();
 850    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 851       Cache.HeaderP->MaxDescFileSize = DF->Size;
 852    Cache.HeaderP->DescFileCount++;
 853
 854    return true;
 855 }
 856                                                                         /*}}}*/
 857 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 858 // ---------------------------------------------------------------------
 859 /* This puts a description structure in the linked list */
 860 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 861                                             const string &Lang,
 862                                             const MD5SumValue &md5sum,
 863                                             map_ptrloc idxmd5str)
 864 {
 865    // Get a structure
 866    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 867    if (Description == 0)
 868       return 0;
 869
 870    // Fill it in
 871    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 872    Desc->ID = Cache.HeaderP->DescriptionCount++;
 873    map_ptrloc const idxlanguage_code = WriteUniqString(Lang);
 874    if (unlikely(idxlanguage_code == 0))
 875       return 0;
 876    Desc->language_code = idxlanguage_code;
 877
 878    if (idxmd5str != 0)
 879       Desc->md5sum = idxmd5str;
 880    else
 881    {
 882       map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 883       if (unlikely(idxmd5sum == 0))
 884          return 0;
 885       Desc->md5sum = idxmd5sum;
 886    }
 887
 888    return Description;
 889 }
 890                                                                         /*}}}*/
 891 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 892 // ---------------------------------------------------------------------
 893 /* This creates a dependency element in the tree. It is linked to the
 894    version and to the package that it is pointing to. */
 895 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 896                                    pkgCache::VerIterator &Ver,
 897                                    string const &Version,
 898                                    unsigned int const &Op,
 899                                    unsigned int const &Type,
 900                                    map_ptrloc* &OldDepLast)
 901 {
 902    map_ptrloc index = 0;
 903    if (Version.empty() == false)
 904    {
 905       int const CmpOp = Op & 0x0F;
 906       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 907       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 908          index = Ver->VerStr;
 909
 910       if (index == 0)
 911       {
 912          void const * const oldMap = Map.Data();
 913          index = WriteStringInMap(Version);
 914          if (unlikely(index == 0))
 915             return false;
 916          if (OldDepLast != 0 && oldMap != Map.Data())
 917             OldDepLast += (map_ptrloc const * const) Map.Data() - (map_ptrloc const * const) oldMap;
 918       }
 919    }
 920    return NewDepends(Pkg, Ver, index, Op, Type, OldDepLast);
 921 }
 922 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 923                                    pkgCache::VerIterator &Ver,
 924                                    map_ptrloc const Version,
 925                                    unsigned int const &Op,
 926                                    unsigned int const &Type,
 927                                    map_ptrloc* &OldDepLast)
 928 {
 929    void const * const oldMap = Map.Data();
 930    // Get a structure
 931    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 932    if (unlikely(Dependency == 0))
 933       return false;
 934
 935    // Fill it in
 936    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 937    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 938    Dep->ParentVer = Ver.Index();
 939    Dep->Type = Type;
 940    Dep->CompareOp = Op;
 941    Dep->Version = Version;
 942    Dep->ID = Cache.HeaderP->DependsCount++;
 943
 944    // Link it to the package
 945    Dep->Package = Pkg.Index();
 946    Dep->NextRevDepends = Pkg->RevDepends;
 947    Pkg->RevDepends = Dep.Index();
 948
 949    // Do we know where to link the Dependency to?
 950    if (OldDepLast == NULL)
 951    {
 952       OldDepLast = &Ver->DependsList;
 953       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 954          OldDepLast = &D->NextDepends;
 955    } else if (oldMap != Map.Data())
 956       OldDepLast += (map_ptrloc const * const) Map.Data() - (map_ptrloc const * const) oldMap;
 957
 958    Dep->NextDepends = *OldDepLast;
 959    *OldDepLast = Dep.Index();
 960    OldDepLast = &Dep->NextDepends;
 961
 962    return true;
 963 }
 964                                                                         /*}}}*/
 965 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 966 // ---------------------------------------------------------------------
 967 /* This creates a Group and the Package to link this dependency to if
 968    needed and handles also the caching of the old endpoint */
 969 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 970                                                const string &PackageName,
 971                                                const string &Arch,
 972                                                const string &Version,
 973                                                unsigned int Op,
 974                                                unsigned int Type)
 975 {
 976    pkgCache::GrpIterator Grp;
 977    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 978    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 979       return false;
 980
 981    // Locate the target package
 982    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 983    // we don't create 'none' packages and their dependencies if we can avoid it …
 984    if (Pkg.end() == true && Arch == "none" && strcmp(Ver.ParentPkg().Arch(), "none") != 0)
 985       return true;
 986    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 987    if (Pkg.end() == true) {
 988       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 989          return false;
 990    }
 991
 992    // Is it a file dependency?
 993    if (unlikely(PackageName[0] == '/'))
 994       FoundFileDeps = true;
 995
 996    /* Caching the old end point speeds up generation substantially */
 997    if (OldDepVer != Ver) {
 998       OldDepLast = NULL;
 999       OldDepVer = Ver;
1000    }
1001
1002    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
1003 }
1004                                                                         /*}}}*/
1005 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1006 // ---------------------------------------------------------------------
1007 /* */
1008 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
1009                                                 const string &PkgName,
1010                                                 const string &PkgArch,
1011                                                 const string &Version)
1012 {
1013    pkgCache &Cache = Owner->Cache;
1014
1015    // We do not add self referencing provides
1016    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1017         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1018       return true;
1019
1020    // Get a structure
1021    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
1022    if (unlikely(Provides == 0))
1023       return false;
1024    Cache.HeaderP->ProvidesCount++;
1025
1026    // Fill it in
1027    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1028    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
1029    Prv->Version = Ver.Index();
1030    Prv->NextPkgProv = Ver->ProvidesList;
1031    Ver->ProvidesList = Prv.Index();
1032    if (Version.empty() == false) {
1033       map_ptrloc const idxProvideVersion = WriteString(Version);
1034       Prv->ProvideVersion = idxProvideVersion;
1035       if (unlikely(idxProvideVersion == 0))
1036          return false;
1037    }
1038
1039    // Locate the target package
1040    pkgCache::PkgIterator Pkg;
1041    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1042    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1043       return false;
1044
1045    // Link it to the package
1046    Prv->ParentPkg = Pkg.Index();
1047    Prv->NextProvides = Pkg->ProvidesList;
1048    Pkg->ProvidesList = Prv.Index();
1049
1050    return true;
1051 }
1052                                                                         /*}}}*/
1053 bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1054       pkgCache::VerIterator const &Ver)
1055 {
1056    return Hash == Ver->Hash;
1057 }
1058                                                                         /*}}}*/
1059 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1060 // ---------------------------------------------------------------------
1061 /* This is used to select which file is to be associated with all newly
1062    added versions. The caller is responsible for setting the IMS fields. */
1063 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
1064                                    const pkgIndexFile &Index,
1065                                    unsigned long Flags)
1066 {
1067    // Get some space for the structure
1068    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
1069    if (unlikely(idxFile == 0))
1070       return false;
1071    CurrentFile = Cache.PkgFileP + idxFile;
1072
1073    // Fill it in
1074    map_ptrloc const idxFileName = WriteStringInMap(File);
1075    map_ptrloc const idxSite = WriteUniqString(Site);
1076    if (unlikely(idxFileName == 0 || idxSite == 0))
1077       return false;
1078    CurrentFile->FileName = idxFileName;
1079    CurrentFile->Site = idxSite;
1080    CurrentFile->NextFile = Cache.HeaderP->FileList;
1081    CurrentFile->Flags = Flags;
1082    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1083    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
1084    if (unlikely(idxIndexType == 0))
1085       return false;
1086    CurrentFile->IndexType = idxIndexType;
1087    PkgFileName = File;
1088    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1089    Cache.HeaderP->PackageFileCount++;
1090
1091    if (Progress != 0)
1092       Progress->SubProgress(Index.Size());
1093    return true;
1094 }
1095                                                                         /*}}}*/
1096 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1097 // ---------------------------------------------------------------------
1098 /* This is used to create handles to strings. Given the same text it
1099    always returns the same number */
1100 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
1101                                                  unsigned int Size)
1102 {
1103    /* We use a very small transient hash table here, this speeds up generation
1104       by a fair amount on slower machines */
1105    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
1106    if (Bucket != 0 &&
1107        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
1108       return Bucket->String;
1109
1110    // Search for an insertion point
1111    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
1112    int Res = 1;
1113    map_ptrloc *Last = &Cache.HeaderP->StringList;
1114    for (; I != Cache.StringItemP; Last = &I->NextItem,
1115         I = Cache.StringItemP + I->NextItem)
1116    {
1117       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
1118       if (Res >= 0)
1119          break;
1120    }
1121
1122    // Match
1123    if (Res == 0)
1124    {
1125       Bucket = I;
1126       return I->String;
1127    }
1128
1129    // Get a structure
1130    void const * const oldMap = Map.Data();
1131    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
1132    if (Item == 0)
1133       return 0;
1134
1135    map_ptrloc const idxString = WriteStringInMap(S,Size);
1136    if (unlikely(idxString == 0))
1137       return 0;
1138    if (oldMap != Map.Data()) {
1139       Last += (map_ptrloc const * const) Map.Data() - (map_ptrloc const * const) oldMap;
1140       I += (pkgCache::StringItem const * const) Map.Data() - (pkgCache::StringItem const * const) oldMap;
1141    }
1142    *Last = Item;
1143
1144    // Fill in the structure
1145    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
1146    ItemP->NextItem = I - Cache.StringItemP;
1147    ItemP->String = idxString;
1148
1149    Bucket = ItemP;
1150    return ItemP->String;
1151 }
1152                                                                         /*}}}*/
1153 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1154 // ---------------------------------------------------------------------
1155 /* This just verifies that each file in the list of index files exists,
1156    has matching attributes with the cache and the cache does not have
1157    any extra files. */
1158 static bool CheckValidity(const string &CacheFile,
1159                           pkgSourceList &List,
1160                           FileIterator Start,
1161                           FileIterator End,
1162                           MMap **OutMap = 0)
1163 {
1164    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1165    // No file, certainly invalid
1166    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1167    {
1168       if (Debug == true)
1169          std::clog << "CacheFile doesn't exist" << std::endl;
1170       return false;
1171    }
1172
1173    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1174    {
1175       if (Debug == true)
1176          std::clog << "sources.list is newer than the cache" << std::endl;
1177       return false;
1178    }
1179
1180    // Map it
1181    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1182    SPtr<MMap> Map = new MMap(CacheF,0);
1183    pkgCache Cache(Map);
1184    if (_error->PendingError() == true || Map->Size() == 0)
1185    {
1186       if (Debug == true)
1187          std::clog << "Errors are pending or Map is empty()" << std::endl;
1188       _error->Discard();
1189       return false;
1190    }
1191
1192    /* Now we check every index file, see if it is in the cache,
1193       verify the IMS data and check that it is on the disk too.. */
1194    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1195    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1196    for (; Start != End; ++Start)
1197    {
1198       if (Debug == true)
1199          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
1200       if ((*Start)->HasPackages() == false)
1201       {
1202          if (Debug == true)
1203             std::clog << "Has NO packages" << std::endl;
1204          continue;
1205       }
1206
1207       if ((*Start)->Exists() == false)
1208       {
1209 #if 0 // mvo: we no longer give a message here (Default Sources spec)
1210          _error->WarningE("stat",_("Couldn't stat source package list %s"),
1211                           (*Start)->Describe().c_str());
1212 #endif
1213          if (Debug == true)
1214             std::clog << "file doesn't exist" << std::endl;
1215          continue;
1216       }
1217
1218       // FindInCache is also expected to do an IMS check.
1219       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
1220       if (File.end() == true)
1221       {
1222          if (Debug == true)
1223             std::clog << "FindInCache returned end-Pointer" << std::endl;
1224          return false;
1225       }
1226
1227       Visited[File->ID] = true;
1228       if (Debug == true)
1229          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1230    }
1231
1232    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1233       if (Visited[I] == false)
1234       {
1235          if (Debug == true)
1236             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1237          return false;
1238       }
1239
1240    if (_error->PendingError() == true)
1241    {
1242       if (Debug == true)
1243       {
1244          std::clog << "Validity failed because of pending errors:" << std::endl;
1245          _error->DumpErrors();
1246       }
1247       _error->Discard();
1248       return false;
1249    }
1250
1251    if (OutMap != 0)
1252       *OutMap = Map.UnGuard();
1253    return true;
1254 }
1255                                                                         /*}}}*/
1256 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1257 // ---------------------------------------------------------------------
1258 /* Size is kind of an abstract notion that is only used for the progress
1259    meter */
1260 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1261 {
1262    unsigned long TotalSize = 0;
1263    for (; Start < End; ++Start)
1264    {
1265       if ((*Start)->HasPackages() == false)
1266          continue;
1267       TotalSize += (*Start)->Size();
1268    }
1269    return TotalSize;
1270 }
1271                                                                         /*}}}*/
1272 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1273 // ---------------------------------------------------------------------
1274 /* */
1275 static bool BuildCache(pkgCacheGenerator &Gen,
1276                        OpProgress *Progress,
1277                        unsigned long &CurrentSize,unsigned long TotalSize,
1278                        FileIterator Start, FileIterator End)
1279 {
1280    FileIterator I;
1281    for (I = Start; I != End; ++I)
1282    {
1283       if ((*I)->HasPackages() == false)
1284          continue;
1285
1286       if ((*I)->Exists() == false)
1287          continue;
1288
1289       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1290       {
1291          _error->Warning("Duplicate sources.list entry %s",
1292                          (*I)->Describe().c_str());
1293          continue;
1294       }
1295
1296       unsigned long Size = (*I)->Size();
1297       if (Progress != NULL)
1298          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1299       CurrentSize += Size;
1300
1301       if ((*I)->Merge(Gen,Progress) == false)
1302          return false;
1303    }
1304
1305    if (Gen.HasFileDeps() == true)
1306    {
1307       if (Progress != NULL)
1308          Progress->Done();
1309       TotalSize = ComputeSize(Start, End);
1310       CurrentSize = 0;
1311       for (I = Start; I != End; ++I)
1312       {
1313          unsigned long Size = (*I)->Size();
1314          if (Progress != NULL)
1315             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1316          CurrentSize += Size;
1317          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1318             return false;
1319       }
1320    }
1321
1322    return true;
1323 }
1324                                                                         /*}}}*/
1325 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1326 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1327    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1328    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1329    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1330    Flags |= MMap::Moveable;
1331    if (_config->FindB("APT::Cache-Fallback", false) == true)
1332       Flags |= MMap::Fallback;
1333    if (CacheF != NULL)
1334       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1335    else
1336       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1337 }
1338                                                                         /*}}}*/
1339 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1340 // ---------------------------------------------------------------------
1341 /* This makes sure that the status cache (the cache that has all
1342    index files from the sources list and all local ones) is ready
1343    to be mmaped. If OutMap is not zero then a MMap object representing
1344    the cache will be stored there. This is pretty much mandetory if you
1345    are using AllowMem. AllowMem lets the function be run as non-root
1346    where it builds the cache 'fast' into a memory buffer. */
1347 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1348                         MMap **OutMap, bool AllowMem)
1349    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1350 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1351                         MMap **OutMap,bool AllowMem)
1352 {
1353    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1354
1355    std::vector<pkgIndexFile *> Files;
1356    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1357         i != List.end();
1358         ++i)
1359    {
1360       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1361       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1362            j != Indexes->end();
1363            ++j)
1364          Files.push_back (*j);
1365    }
1366
1367    unsigned long const EndOfSource = Files.size();
1368    if (_system->AddStatusFiles(Files) == false)
1369       return false;
1370
1371    // Decide if we can write to the files..
1372    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1373    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1374
1375    // ensure the cache directory exists
1376    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1377    {
1378       string dir = _config->FindDir("Dir::Cache");
1379       size_t const len = dir.size();
1380       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1381          dir = dir.substr(0, len - 5);
1382       if (CacheFile.empty() == false)
1383          CreateDirectory(dir, flNotFile(CacheFile));
1384       if (SrcCacheFile.empty() == false)
1385          CreateDirectory(dir, flNotFile(SrcCacheFile));
1386    }
1387
1388    // Decide if we can write to the cache
1389    bool Writeable = false;
1390    if (CacheFile.empty() == false)
1391       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1392    else
1393       if (SrcCacheFile.empty() == false)
1394          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1395    if (Debug == true)
1396       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1397
1398    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1399       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1400
1401    if (Progress != NULL)
1402       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1403
1404    // Cache is OK, Fin.
1405    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1406    {
1407       if (Progress != NULL)
1408          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1409       if (Debug == true)
1410          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1411       return true;
1412    }
1413    else if (Debug == true)
1414          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1415
1416    /* At this point we know we need to reconstruct the package cache,
1417       begin. */
1418    SPtr<FileFd> CacheF;
1419    SPtr<DynamicMMap> Map;
1420    if (Writeable == true && CacheFile.empty() == false)
1421    {
1422       _error->PushToStack();
1423       unlink(CacheFile.c_str());
1424       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1425       fchmod(CacheF->Fd(),0644);
1426       Map = CreateDynamicMMap(CacheF, MMap::Public);
1427       if (_error->PendingError() == true)
1428       {
1429          delete CacheF.UnGuard();
1430          delete Map.UnGuard();
1431          if (Debug == true)
1432             std::clog << "Open filebased MMap FAILED" << std::endl;
1433          Writeable = false;
1434          if (AllowMem == false)
1435          {
1436             _error->MergeWithStack();
1437             return false;
1438          }
1439          _error->RevertToStack();
1440       }
1441       else
1442       {
1443          _error->MergeWithStack();
1444          if (Debug == true)
1445             std::clog << "Open filebased MMap" << std::endl;
1446       }
1447    }
1448    if (Writeable == false || CacheFile.empty() == true)
1449    {
1450       // Just build it in memory..
1451       Map = CreateDynamicMMap(NULL);
1452       if (Debug == true)
1453          std::clog << "Open memory Map (not filebased)" << std::endl;
1454    }
1455
1456    // Lets try the source cache.
1457    unsigned long CurrentSize = 0;
1458    unsigned long TotalSize = 0;
1459    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1460                      Files.begin()+EndOfSource) == true)
1461    {
1462       if (Debug == true)
1463          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1464       // Preload the map with the source cache
1465       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1466       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1467       if ((alloc == 0 && _error->PendingError())
1468                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1469                                 SCacheF.Size()) == false)
1470          return false;
1471
1472       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1473
1474       // Build the status cache
1475       pkgCacheGenerator Gen(Map.Get(),Progress);
1476       if (_error->PendingError() == true)
1477          return false;
1478       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1479                      Files.begin()+EndOfSource,Files.end()) == false)
1480          return false;
1481    }
1482    else
1483    {
1484       if (Debug == true)
1485          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1486       TotalSize = ComputeSize(Files.begin(),Files.end());
1487
1488       // Build the source cache
1489       pkgCacheGenerator Gen(Map.Get(),Progress);
1490       if (_error->PendingError() == true)
1491          return false;
1492       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1493                      Files.begin(),Files.begin()+EndOfSource) == false)
1494          return false;
1495
1496       // Write it back
1497       if (Writeable == true && SrcCacheFile.empty() == false)
1498       {
1499          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1500          if (_error->PendingError() == true)
1501             return false;
1502
1503          fchmod(SCacheF.Fd(),0644);
1504
1505          // Write out the main data
1506          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1507             return _error->Error(_("IO Error saving source cache"));
1508          SCacheF.Sync();
1509
1510          // Write out the proper header
1511          Gen.GetCache().HeaderP->Dirty = false;
1512          if (SCacheF.Seek(0) == false ||
1513              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1514             return _error->Error(_("IO Error saving source cache"));
1515          Gen.GetCache().HeaderP->Dirty = true;
1516          SCacheF.Sync();
1517       }
1518
1519       // Build the status cache
1520       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1521                      Files.begin()+EndOfSource,Files.end()) == false)
1522          return false;
1523    }
1524    if (Debug == true)
1525       std::clog << "Caches are ready for shipping" << std::endl;
1526
1527    if (_error->PendingError() == true)
1528       return false;
1529    if (OutMap != 0)
1530    {
1531       if (CacheF != 0)
1532       {
1533          delete Map.UnGuard();
1534          *OutMap = new MMap(*CacheF,0);
1535       }
1536       else
1537       {
1538          *OutMap = Map.UnGuard();
1539       }
1540    }
1541
1542    return true;
1543 }
1544                                                                         /*}}}*/
1545 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1546 // ---------------------------------------------------------------------
1547 /* */
1548 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1549    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1550 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1551 {
1552    std::vector<pkgIndexFile *> Files;
1553    unsigned long EndOfSource = Files.size();
1554    if (_system->AddStatusFiles(Files) == false)
1555       return false;
1556
1557    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1558    unsigned long CurrentSize = 0;
1559    unsigned long TotalSize = 0;
1560
1561    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1562
1563    // Build the status cache
1564    if (Progress != NULL)
1565       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1566    pkgCacheGenerator Gen(Map.Get(),Progress);
1567    if (_error->PendingError() == true)
1568       return false;
1569    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1570                   Files.begin()+EndOfSource,Files.end()) == false)
1571       return false;
1572
1573    if (_error->PendingError() == true)
1574       return false;
1575    *OutMap = Map.UnGuard();
1576
1577    return true;
1578 }
1579                                                                         /*}}}*/
1580 // IsDuplicateDescription                                               /*{{{*/
1581 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1582                             MD5SumValue const &CurMd5, std::string const &CurLang)
1583 {
1584    // Descriptions in the same link-list have all the same md5
1585    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1586       return false;
1587    for (; Desc.end() == false; ++Desc)
1588       if (Desc.LanguageCode() == CurLang)
1589          return true;
1590    return false;
1591 }
1592                                                                         /*}}}*/
1593 // CacheGenerator::FinishCache                                          /*{{{*/
1594 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1595 {
1596    return true;
1597 }
1598                                                                         /*}}}*/