apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  45 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  46
  47 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  48                             MD5SumValue const &CurMd5, std::string const &CurLang);
  49
  50 using std::string;
  51
  52 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We set the dirty flag and make sure that is written to the disk */
  55 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  56                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  57                     FoundFileDeps(0)
  58 {
  59    CurrentFile = 0;
  60    memset(UniqHash,0,sizeof(UniqHash));
  61
  62    if (_error->PendingError() == true)
  63       return;
  64
  65    if (Map.Size() == 0)
  66    {
  67       // Setup the map interface..
  68       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  69       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->HashTableSize * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       // this pointer is set in ReMap, but we need it now for WriteUniqString
  86       Cache.StringItemP = (pkgCache::StringItem *)Map.Data();
  87       map_stringitem_t const idxArchitecture = WriteUniqString(_config->Find("APT::Architecture"));
  88       if (unlikely(idxArchitecture == 0))
  89          return;
  90       Cache.HeaderP->Architecture = idxArchitecture;
  91
  92       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  93       if (archs.size() > 1)
  94       {
  95          std::vector<std::string>::const_iterator a = archs.begin();
  96          std::string list = *a;
  97          for (++a; a != archs.end(); ++a)
  98             list.append(",").append(*a);
  99          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
 100          if (unlikely(idxArchitectures == 0))
 101             return;
 102          Cache.HeaderP->Architectures = idxArchitectures;
 103       }
 104       else
 105          Cache.HeaderP->Architectures = idxArchitecture;
 106
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Map.Size();
 138    Map.Sync(0,sizeof(pkgCache::Header));
 139 }
 140                                                                         /*}}}*/
 141 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 142    if (oldMap == newMap)
 143       return;
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 147
 148    Cache.ReMap(false);
 149
 150    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 151
 152    for (size_t i = 0; i < _count(UniqHash); ++i)
 153       if (UniqHash[i] != 0)
 154          UniqHash[i] += (pkgCache::StringItem const * const) newMap - (pkgCache::StringItem const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177 }                                                                       /*}}}*/
 178 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 179 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 180                                         const unsigned long &Len) {
 181    void const * const oldMap = Map.Data();
 182    map_stringitem_t const index = Map.WriteString(String, Len);
 183    if (index != 0)
 184       ReMap(oldMap, Map.Data());
 185    return index;
 186 }
 187                                                                         /*}}}*/
 188 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 189 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 190    void const * const oldMap = Map.Data();
 191    map_stringitem_t const index = Map.WriteString(String);
 192    if (index != 0)
 193       ReMap(oldMap, Map.Data());
 194    return index;
 195 }
 196                                                                         /*}}}*/
 197 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 198    void const * const oldMap = Map.Data();
 199    map_pointer_t const index = Map.Allocate(size);
 200    if (index != 0)
 201       ReMap(oldMap, Map.Data());
 202    return index;
 203 }
 204                                                                         /*}}}*/
 205 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 206 // ---------------------------------------------------------------------
 207 /* This provides the generation of the entries in the cache. Each loop
 208    goes through a single package record from the underlying parse engine. */
 209 bool pkgCacheGenerator::MergeList(ListParser &List,
 210                                   pkgCache::VerIterator *OutVer)
 211 {
 212    List.Owner = this;
 213
 214    unsigned int Counter = 0;
 215    while (List.Step() == true)
 216    {
 217       string const PackageName = List.Package();
 218       if (PackageName.empty() == true)
 219          return false;
 220
 221       Counter++;
 222       if (Counter % 100 == 0 && Progress != 0)
 223          Progress->Progress(List.Offset());
 224
 225       string Arch = List.Architecture();
 226       string const Version = List.Version();
 227       if (Version.empty() == true && Arch.empty() == true)
 228       {
 229          // package descriptions
 230          if (MergeListGroup(List, PackageName) == false)
 231             return false;
 232          continue;
 233       }
 234
 235       if (Arch.empty() == true)
 236       {
 237          // use the pseudo arch 'none' for arch-less packages
 238          Arch = "none";
 239          /* We might built a SingleArchCache here, which we don't want to blow up
 240             just for these :none packages to a proper MultiArchCache, so just ensure
 241             that we have always a native package structure first for SingleArch */
 242          pkgCache::PkgIterator NP;
 243          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 244          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 245          // TRANSLATOR: The first placeholder is a package name,
 246          // the other two should be copied verbatim as they include debug info
 247          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 248                               PackageName.c_str(), "NewPackage", 0);
 249       }
 250
 251       // Get a pointer to the package structure
 252       pkgCache::PkgIterator Pkg;
 253       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 254       if (NewPackage(Pkg, PackageName, Arch) == false)
 255          // TRANSLATOR: The first placeholder is a package name,
 256          // the other two should be copied verbatim as they include debug info
 257          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 258                               PackageName.c_str(), "NewPackage", 1);
 259
 260
 261       if (Version.empty() == true)
 262       {
 263          if (MergeListPackage(List, Pkg) == false)
 264             return false;
 265       }
 266       else
 267       {
 268          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 269             return false;
 270       }
 271
 272       if (OutVer != 0)
 273       {
 274          FoundFileDeps |= List.HasFileDeps();
 275          return true;
 276       }
 277    }
 278
 279    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 280       return _error->Error(_("Wow, you exceeded the number of package "
 281                              "names this APT is capable of."));
 282    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 283       return _error->Error(_("Wow, you exceeded the number of versions "
 284                              "this APT is capable of."));
 285    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 286       return _error->Error(_("Wow, you exceeded the number of descriptions "
 287                              "this APT is capable of."));
 288    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 289       return _error->Error(_("Wow, you exceeded the number of dependencies "
 290                              "this APT is capable of."));
 291
 292    FoundFileDeps |= List.HasFileDeps();
 293    return true;
 294 }
 295 // CacheGenerator::MergeListGroup                                       /*{{{*/
 296 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 297 {
 298    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 299    // a group has no data on it's own, only packages have it but these
 300    // stanzas like this come from Translation- files to add descriptions,
 301    // but without a version we don't need a description for it…
 302    if (Grp.end() == true)
 303       return true;
 304    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 305
 306    pkgCache::PkgIterator Pkg;
 307    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 308    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 309       if (MergeListPackage(List, Pkg) == false)
 310          return false;
 311
 312    return true;
 313 }
 314                                                                         /*}}}*/
 315 // CacheGenerator::MergeListPackage                                     /*{{{*/
 316 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 317 {
 318    // we first process the package, then the descriptions
 319    // (for deb this package processing is in fact a no-op)
 320    pkgCache::VerIterator Ver(Cache);
 321    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 322    if (List.UsePackage(Pkg, Ver) == false)
 323       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 324                            Pkg.Name(), "UsePackage", 1);
 325
 326    // Find the right version to write the description
 327    MD5SumValue CurMd5 = List.Description_md5();
 328    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 329       return true;
 330    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 331    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 332    {
 333       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 334
 335       // a version can only have one md5 describing it
 336       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 337          continue;
 338
 339       map_stringitem_t md5idx = VerDesc->md5sum;
 340       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 341       {
 342          // don't add a new description if we have one for the given
 343          // md5 && language
 344          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 345             continue;
 346
 347          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 348       }
 349
 350       // we can stop here as all "same" versions will share the description
 351       break;
 352    }
 353
 354    return true;
 355 }
 356                                                                         /*}}}*/
 357 // CacheGenerator::MergeListVersion                                     /*{{{*/
 358 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 359                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 360 {
 361    pkgCache::VerIterator Ver = Pkg.VersionList();
 362    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 363    map_pointer_t *LastVer = &Pkg->VersionList;
 364    void const * oldMap = Map.Data();
 365
 366    unsigned short const Hash = List.VersionHash();
 367    if (Ver.end() == false)
 368    {
 369       /* We know the list is sorted so we use that fact in the search.
 370          Insertion of new versions is done with correct sorting */
 371       int Res = 1;
 372       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 373       {
 374          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 375          // Version is higher as current version - insert here
 376          if (Res > 0)
 377             break;
 378          // Versionstrings are equal - is hash also equal?
 379          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 380             break;
 381          // proceed with the next till we have either the right
 382          // or we found another version (which will be lower)
 383       }
 384
 385       /* We already have a version for this item, record that we saw it */
 386       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 387       {
 388          if (List.UsePackage(Pkg,Ver) == false)
 389             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 390                                  Pkg.Name(), "UsePackage", 2);
 391
 392          if (NewFileVer(Ver,List) == false)
 393             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 394                                  Pkg.Name(), "NewFileVer", 1);
 395
 396          // Read only a single record and return
 397          if (OutVer != 0)
 398          {
 399             *OutVer = Ver;
 400             return true;
 401          }
 402
 403          return true;
 404       }
 405    }
 406
 407    // Add a new version
 408    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 409    if (verindex == 0 && _error->PendingError())
 410       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 411                            Pkg.Name(), "NewVersion", 1);
 412
 413    if (oldMap != Map.Data())
 414          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 415    *LastVer = verindex;
 416
 417    if (unlikely(List.NewVersion(Ver) == false))
 418       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                            Pkg.Name(), "NewVersion", 2);
 420
 421    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 422       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 423                            Pkg.Name(), "UsePackage", 3);
 424
 425    if (unlikely(NewFileVer(Ver,List) == false))
 426       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 427                            Pkg.Name(), "NewFileVer", 2);
 428
 429    pkgCache::GrpIterator Grp = Pkg.Group();
 430    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 431
 432    /* If it is the first version of this package we need to add implicit
 433       Multi-Arch dependencies to all other package versions in the group now -
 434       otherwise we just add them for this new version */
 435    if (Pkg.VersionList()->NextVer == 0)
 436    {
 437       pkgCache::PkgIterator P = Grp.PackageList();
 438       Dynamic<pkgCache::PkgIterator> DynP(P);
 439       for (; P.end() != true; P = Grp.NextPkg(P))
 440       {
 441          if (P->ID == Pkg->ID)
 442             continue;
 443          pkgCache::VerIterator V = P.VersionList();
 444          Dynamic<pkgCache::VerIterator> DynV(V);
 445          for (; V.end() != true; ++V)
 446             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 447                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 448                                     Pkg.Name(), "AddImplicitDepends", 1);
 449       }
 450       /* :none packages are packages without an architecture. They are forbidden by
 451          debian-policy, so usually they will only be in (old) dpkg status files -
 452          and dpkg will complain about them - and are pretty rare. We therefore do
 453          usually not create conflicts while the parent is created, but only if a :none
 454          package (= the target) appears. This creates incorrect dependencies on :none
 455          for architecture-specific dependencies on the package we copy from, but we
 456          will ignore this bug as architecture-specific dependencies are only allowed
 457          in jessie and until then the :none packages should be extinct (hopefully).
 458          In other words: This should work long enough to allow graceful removal of
 459          these packages, it is not supposed to allow users to keep using them … */
 460       if (strcmp(Pkg.Arch(), "none") == 0)
 461       {
 462          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 463          if (M.end() == false && Pkg != M)
 464          {
 465             pkgCache::DepIterator D = M.RevDependsList();
 466             Dynamic<pkgCache::DepIterator> DynD(D);
 467             for (; D.end() == false; ++D)
 468             {
 469                if ((D->Type != pkgCache::Dep::Conflicts &&
 470                     D->Type != pkgCache::Dep::DpkgBreaks &&
 471                     D->Type != pkgCache::Dep::Replaces) ||
 472                    D.ParentPkg().Group() == Grp)
 473                   continue;
 474
 475                map_pointer_t *OldDepLast = NULL;
 476                pkgCache::VerIterator ConVersion = D.ParentVer();
 477                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 478                // duplicate the Conflicts/Breaks/Replaces for :none arch
 479                NewDepends(Pkg, ConVersion, D->Version,
 480                      D->CompareOp, D->Type, OldDepLast);
 481             }
 482          }
 483       }
 484    }
 485    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 486       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 487                            Pkg.Name(), "AddImplicitDepends", 2);
 488
 489    // Read only a single record and return
 490    if (OutVer != 0)
 491    {
 492       *OutVer = Ver;
 493       return true;
 494    }
 495
 496    /* Record the Description(s) based on their master md5sum */
 497    MD5SumValue CurMd5 = List.Description_md5();
 498    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 499       return true;
 500
 501    /* Before we add a new description we first search in the group for
 502       a version with a description of the same MD5 - if so we reuse this
 503       description group instead of creating our own for this version */
 504    for (pkgCache::PkgIterator P = Grp.PackageList();
 505         P.end() == false; P = Grp.NextPkg(P))
 506    {
 507       for (pkgCache::VerIterator V = P.VersionList();
 508            V.end() == false; ++V)
 509       {
 510          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 511             continue;
 512          Ver->DescriptionList = V->DescriptionList;
 513       }
 514    }
 515
 516    // We haven't found reusable descriptions, so add the first description(s)
 517    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 518    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 519    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 520       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 521          return false;
 522    return true;
 523 }
 524                                                                         /*}}}*/
 525 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 526 {
 527    pkgCache::DescIterator Desc;
 528    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 529
 530    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 531    if (unlikely(descindex == 0 && _error->PendingError()))
 532       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 533             Ver.ParentPkg().Name(), "NewDescription", 1);
 534
 535    md5idx = Desc->md5sum;
 536    Desc->ParentPkg = Ver.ParentPkg().Index();
 537
 538    // we add at the end, so that the start is constant as we need
 539    // that to be able to efficiently share these lists
 540    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 541    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 542    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 543    *LastNextDesc = descindex;
 544
 545    if (NewFileDesc(Desc,List) == false)
 546       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 547             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 548
 549    return true;
 550 }
 551                                                                         /*}}}*/
 552                                                                         /*}}}*/
 553 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 554 // ---------------------------------------------------------------------
 555 /* If we found any file depends while parsing the main list we need to
 556    resolve them. Since it is undesired to load the entire list of files
 557    into the cache as virtual packages we do a two stage effort. MergeList
 558    identifies the file depends and this creates Provdies for them by
 559    re-parsing all the indexs. */
 560 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 561 {
 562    List.Owner = this;
 563
 564    unsigned int Counter = 0;
 565    while (List.Step() == true)
 566    {
 567       string PackageName = List.Package();
 568       if (PackageName.empty() == true)
 569          return false;
 570       string Version = List.Version();
 571       if (Version.empty() == true)
 572          continue;
 573
 574       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 575       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 576       if (Pkg.end() == true)
 577          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 578                                 PackageName.c_str(), "FindPkg", 1);
 579       Counter++;
 580       if (Counter % 100 == 0 && Progress != 0)
 581          Progress->Progress(List.Offset());
 582
 583       unsigned short Hash = List.VersionHash();
 584       pkgCache::VerIterator Ver = Pkg.VersionList();
 585       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 586       for (; Ver.end() == false; ++Ver)
 587       {
 588          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 589          {
 590             if (List.CollectFileProvides(Cache,Ver) == false)
 591                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 592                                     PackageName.c_str(), "CollectFileProvides", 1);
 593             break;
 594          }
 595       }
 596
 597       if (Ver.end() == true)
 598          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 599    }
 600
 601    return true;
 602 }
 603                                                                         /*}}}*/
 604 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 605 // ---------------------------------------------------------------------
 606 /* This creates a new group structure and adds it to the hash table */
 607 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 608 {
 609    Grp = Cache.FindGrp(Name);
 610    if (Grp.end() == false)
 611       return true;
 612
 613    // Get a structure
 614    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 615    if (unlikely(Group == 0))
 616       return false;
 617
 618    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 619    map_pointer_t const idxName = WriteStringInMap(Name);
 620    if (unlikely(idxName == 0))
 621       return false;
 622    Grp->Name = idxName;
 623
 624    // Insert it into the hash table
 625    unsigned long const Hash = Cache.Hash(Name);
 626    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTable()[Hash];
 627    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 628       insertAt = &(Cache.GrpP + *insertAt)->Next;
 629    Grp->Next = *insertAt;
 630    *insertAt = Group;
 631
 632    Grp->ID = Cache.HeaderP->GroupCount++;
 633    return true;
 634 }
 635                                                                         /*}}}*/
 636 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 637 // ---------------------------------------------------------------------
 638 /* This creates a new package structure and adds it to the hash table */
 639 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 640                                         const string &Arch) {
 641    pkgCache::GrpIterator Grp;
 642    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 643    if (unlikely(NewGroup(Grp, Name) == false))
 644       return false;
 645
 646    Pkg = Grp.FindPkg(Arch);
 647       if (Pkg.end() == false)
 648          return true;
 649
 650    // Get a structure
 651    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 652    if (unlikely(Package == 0))
 653       return false;
 654    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 655
 656    // Insert the package into our package list
 657    if (Grp->FirstPackage == 0) // the group is new
 658    {
 659       Grp->FirstPackage = Package;
 660       // Insert it into the hash table
 661       map_id_t const Hash = Cache.Hash(Name);
 662       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTable()[Hash];
 663       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 664          insertAt = &(Cache.PkgP + *insertAt)->Next;
 665       Pkg->Next = *insertAt;
 666       *insertAt = Package;
 667    }
 668    else // Group the Packages together
 669    {
 670       // this package is the new last package
 671       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 672       Pkg->Next = LastPkg->Next;
 673       LastPkg->Next = Package;
 674    }
 675    Grp->LastPackage = Package;
 676
 677    // Set the name, arch and the ID
 678 #if __GNUC__ >= 4
 679         #pragma GCC diagnostic push
 680         #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 681 #endif
 682    Pkg->Name = Grp->Name;
 683 #if __GNUC__ >= 4
 684         #pragma GCC diagnostic pop
 685 #endif
 686    Pkg->Group = Grp.Index();
 687    // all is mapped to the native architecture
 688    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 689    if (unlikely(idxArch == 0))
 690       return false;
 691    Pkg->Arch = idxArch;
 692    Pkg->ID = Cache.HeaderP->PackageCount++;
 693
 694    return true;
 695 }
 696                                                                         /*}}}*/
 697 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 698 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 699                                            pkgCache::PkgIterator &P,
 700                                            pkgCache::VerIterator &V)
 701 {
 702    // copy P.Arch() into a string here as a cache remap
 703    // in NewDepends() later may alter the pointer location
 704    string Arch = P.Arch() == NULL ? "" : P.Arch();
 705    map_pointer_t *OldDepLast = NULL;
 706    /* MultiArch handling introduces a lot of implicit Dependencies:
 707       - MultiArch: same → Co-Installable if they have the same version
 708       - All others conflict with all other group members */
 709    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 710    pkgCache::PkgIterator D = G.PackageList();
 711    Dynamic<pkgCache::PkgIterator> DynD(D);
 712    map_stringitem_t const VerStrIdx = V->VerStr;
 713    for (; D.end() != true; D = G.NextPkg(D))
 714    {
 715       if (Arch == D.Arch() || D->VersionList == 0)
 716          continue;
 717       /* We allow only one installed arch at the time
 718          per group, therefore each group member conflicts
 719          with all other group members */
 720       if (coInstall == true)
 721       {
 722          // Replaces: ${self}:other ( << ${binary:Version})
 723          NewDepends(D, V, VerStrIdx,
 724                     pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 725                     OldDepLast);
 726          // Breaks: ${self}:other (!= ${binary:Version})
 727          NewDepends(D, V, VerStrIdx,
 728                     pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 729                     OldDepLast);
 730       } else {
 731          // Conflicts: ${self}:other
 732          NewDepends(D, V, 0,
 733                     pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 734                     OldDepLast);
 735       }
 736    }
 737    return true;
 738 }
 739 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 740                                            pkgCache::PkgIterator &D)
 741 {
 742    /* MultiArch handling introduces a lot of implicit Dependencies:
 743       - MultiArch: same → Co-Installable if they have the same version
 744       - All others conflict with all other group members */
 745    map_pointer_t *OldDepLast = NULL;
 746    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 747    if (coInstall == true)
 748    {
 749       map_stringitem_t const VerStrIdx = V->VerStr;
 750       // Replaces: ${self}:other ( << ${binary:Version})
 751       NewDepends(D, V, VerStrIdx,
 752                  pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 753                  OldDepLast);
 754       // Breaks: ${self}:other (!= ${binary:Version})
 755       NewDepends(D, V, VerStrIdx,
 756                  pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 757                  OldDepLast);
 758    } else {
 759       // Conflicts: ${self}:other
 760       NewDepends(D, V, 0,
 761                  pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 762                  OldDepLast);
 763    }
 764    return true;
 765 }
 766
 767                                                                         /*}}}*/
 768 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 769 // ---------------------------------------------------------------------
 770 /* */
 771 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 772                                    ListParser &List)
 773 {
 774    if (CurrentFile == 0)
 775       return true;
 776
 777    // Get a structure
 778    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 779    if (VerFile == 0)
 780       return false;
 781
 782    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 783    VF->File = CurrentFile - Cache.PkgFileP;
 784
 785    // Link it to the end of the list
 786    map_pointer_t *Last = &Ver->FileList;
 787    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 788       Last = &V->NextFile;
 789    VF->NextFile = *Last;
 790    *Last = VF.Index();
 791
 792    VF->Offset = List.Offset();
 793    VF->Size = List.Size();
 794    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 795       Cache.HeaderP->MaxVerFileSize = VF->Size;
 796    Cache.HeaderP->VerFileCount++;
 797
 798    return true;
 799 }
 800                                                                         /*}}}*/
 801 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 802 // ---------------------------------------------------------------------
 803 /* This puts a version structure in the linked list */
 804 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 805                                             const string &VerStr,
 806                                             map_pointer_t const ParentPkg,
 807                                             unsigned short const Hash,
 808                                             map_pointer_t const Next)
 809 {
 810    // Get a structure
 811    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 812    if (Version == 0)
 813       return 0;
 814
 815    // Fill it in
 816    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 817    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 818    Ver->NextVer = Next;
 819    Ver->ParentPkg = ParentPkg;
 820    Ver->Hash = Hash;
 821    Ver->ID = Cache.HeaderP->VersionCount++;
 822
 823    // try to find the version string in the group for reuse
 824    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 825    pkgCache::GrpIterator Grp = Pkg.Group();
 826    if (Pkg.end() == false && Grp.end() == false)
 827    {
 828       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 829       {
 830          if (Pkg == P)
 831             continue;
 832          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 833          {
 834             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 835             if (cmp == 0)
 836             {
 837                Ver->VerStr = V->VerStr;
 838                return Version;
 839             }
 840             else if (cmp < 0)
 841                break;
 842          }
 843       }
 844    }
 845    // haven't found the version string, so create
 846    map_stringitem_t const idxVerStr = WriteStringInMap(VerStr);
 847    if (unlikely(idxVerStr == 0))
 848       return 0;
 849    Ver->VerStr = idxVerStr;
 850    return Version;
 851 }
 852                                                                         /*}}}*/
 853 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 854 // ---------------------------------------------------------------------
 855 /* */
 856 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 857                                    ListParser &List)
 858 {
 859    if (CurrentFile == 0)
 860       return true;
 861
 862    // Get a structure
 863    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 864    if (DescFile == 0)
 865       return false;
 866
 867    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 868    DF->File = CurrentFile - Cache.PkgFileP;
 869
 870    // Link it to the end of the list
 871    map_pointer_t *Last = &Desc->FileList;
 872    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 873       Last = &D->NextFile;
 874
 875    DF->NextFile = *Last;
 876    *Last = DF.Index();
 877
 878    DF->Offset = List.Offset();
 879    DF->Size = List.Size();
 880    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 881       Cache.HeaderP->MaxDescFileSize = DF->Size;
 882    Cache.HeaderP->DescFileCount++;
 883
 884    return true;
 885 }
 886                                                                         /*}}}*/
 887 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 888 // ---------------------------------------------------------------------
 889 /* This puts a description structure in the linked list */
 890 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 891                                             const string &Lang,
 892                                             const MD5SumValue &md5sum,
 893                                             map_stringitem_t const idxmd5str)
 894 {
 895    // Get a structure
 896    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 897    if (Description == 0)
 898       return 0;
 899
 900    // Fill it in
 901    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 902    Desc->ID = Cache.HeaderP->DescriptionCount++;
 903    map_stringitem_t const idxlanguage_code = WriteUniqString(Lang);
 904    if (unlikely(idxlanguage_code == 0))
 905       return 0;
 906    Desc->language_code = idxlanguage_code;
 907
 908    if (idxmd5str != 0)
 909       Desc->md5sum = idxmd5str;
 910    else
 911    {
 912       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 913       if (unlikely(idxmd5sum == 0))
 914          return 0;
 915       Desc->md5sum = idxmd5sum;
 916    }
 917
 918    return Description;
 919 }
 920                                                                         /*}}}*/
 921 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 922 // ---------------------------------------------------------------------
 923 /* This creates a dependency element in the tree. It is linked to the
 924    version and to the package that it is pointing to. */
 925 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 926                                    pkgCache::VerIterator &Ver,
 927                                    string const &Version,
 928                                    unsigned int const &Op,
 929                                    unsigned int const &Type,
 930                                    map_stringitem_t* &OldDepLast)
 931 {
 932    map_stringitem_t index = 0;
 933    if (Version.empty() == false)
 934    {
 935       int const CmpOp = Op & 0x0F;
 936       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 937       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 938          index = Ver->VerStr;
 939
 940       if (index == 0)
 941       {
 942          void const * const oldMap = Map.Data();
 943          index = WriteStringInMap(Version);
 944          if (unlikely(index == 0))
 945             return false;
 946          if (OldDepLast != 0 && oldMap != Map.Data())
 947             OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 948       }
 949    }
 950    return NewDepends(Pkg, Ver, index, Op, Type, OldDepLast);
 951 }
 952 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 953                                    pkgCache::VerIterator &Ver,
 954                                    map_pointer_t const Version,
 955                                    unsigned int const &Op,
 956                                    unsigned int const &Type,
 957                                    map_pointer_t* &OldDepLast)
 958 {
 959    void const * const oldMap = Map.Data();
 960    // Get a structure
 961    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 962    if (unlikely(Dependency == 0))
 963       return false;
 964
 965    // Fill it in
 966    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 967    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 968    Dep->ParentVer = Ver.Index();
 969    Dep->Type = Type;
 970    Dep->CompareOp = Op;
 971    Dep->Version = Version;
 972    Dep->ID = Cache.HeaderP->DependsCount++;
 973
 974    // Link it to the package
 975    Dep->Package = Pkg.Index();
 976    Dep->NextRevDepends = Pkg->RevDepends;
 977    Pkg->RevDepends = Dep.Index();
 978
 979    // Do we know where to link the Dependency to?
 980    if (OldDepLast == NULL)
 981    {
 982       OldDepLast = &Ver->DependsList;
 983       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 984          OldDepLast = &D->NextDepends;
 985    } else if (oldMap != Map.Data())
 986       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 987
 988    Dep->NextDepends = *OldDepLast;
 989    *OldDepLast = Dep.Index();
 990    OldDepLast = &Dep->NextDepends;
 991
 992    return true;
 993 }
 994                                                                         /*}}}*/
 995 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 996 // ---------------------------------------------------------------------
 997 /* This creates a Group and the Package to link this dependency to if
 998    needed and handles also the caching of the old endpoint */
 999 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
1000                                                const string &PackageName,
1001                                                const string &Arch,
1002                                                const string &Version,
1003                                                unsigned int Op,
1004                                                unsigned int Type)
1005 {
1006    pkgCache::GrpIterator Grp;
1007    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1008    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1009       return false;
1010
1011    // Locate the target package
1012    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
1013    // we don't create 'none' packages and their dependencies if we can avoid it …
1014    if (Pkg.end() == true && Arch == "none" && strcmp(Ver.ParentPkg().Arch(), "none") != 0)
1015       return true;
1016    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1017    if (Pkg.end() == true) {
1018       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1019          return false;
1020    }
1021
1022    // Is it a file dependency?
1023    if (unlikely(PackageName[0] == '/'))
1024       FoundFileDeps = true;
1025
1026    /* Caching the old end point speeds up generation substantially */
1027    if (OldDepVer != Ver) {
1028       OldDepLast = NULL;
1029       OldDepVer = Ver;
1030    }
1031
1032    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
1033 }
1034                                                                         /*}}}*/
1035 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1036 // ---------------------------------------------------------------------
1037 /* */
1038 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
1039                                                 const string &PkgName,
1040                                                 const string &PkgArch,
1041                                                 const string &Version)
1042 {
1043    pkgCache &Cache = Owner->Cache;
1044
1045    // We do not add self referencing provides
1046    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1047         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1048       return true;
1049
1050    // Get a structure
1051    map_pointer_t const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
1052    if (unlikely(Provides == 0))
1053       return false;
1054    Cache.HeaderP->ProvidesCount++;
1055
1056    // Fill it in
1057    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1058    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
1059    Prv->Version = Ver.Index();
1060    Prv->NextPkgProv = Ver->ProvidesList;
1061    Ver->ProvidesList = Prv.Index();
1062    if (Version.empty() == false) {
1063       map_stringitem_t const idxProvideVersion = WriteString(Version);
1064       Prv->ProvideVersion = idxProvideVersion;
1065       if (unlikely(idxProvideVersion == 0))
1066          return false;
1067    }
1068
1069    // Locate the target package
1070    pkgCache::PkgIterator Pkg;
1071    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1072    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1073       return false;
1074
1075    // Link it to the package
1076    Prv->ParentPkg = Pkg.Index();
1077    Prv->NextProvides = Pkg->ProvidesList;
1078    Pkg->ProvidesList = Prv.Index();
1079
1080    return true;
1081 }
1082                                                                         /*}}}*/
1083 bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1084       pkgCache::VerIterator const &Ver)
1085 {
1086    return Hash == Ver->Hash;
1087 }
1088                                                                         /*}}}*/
1089 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1090 // ---------------------------------------------------------------------
1091 /* This is used to select which file is to be associated with all newly
1092    added versions. The caller is responsible for setting the IMS fields. */
1093 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
1094                                    const pkgIndexFile &Index,
1095                                    unsigned long Flags)
1096 {
1097    // Get some space for the structure
1098    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1099    if (unlikely(idxFile == 0))
1100       return false;
1101    CurrentFile = Cache.PkgFileP + idxFile;
1102
1103    // Fill it in
1104    map_stringitem_t const idxFileName = WriteStringInMap(File);
1105    map_stringitem_t const idxSite = WriteUniqString(Site);
1106    if (unlikely(idxFileName == 0 || idxSite == 0))
1107       return false;
1108    CurrentFile->FileName = idxFileName;
1109    CurrentFile->Site = idxSite;
1110    CurrentFile->NextFile = Cache.HeaderP->FileList;
1111    CurrentFile->Flags = Flags;
1112    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1113    map_stringitem_t const idxIndexType = WriteUniqString(Index.GetType()->Label);
1114    if (unlikely(idxIndexType == 0))
1115       return false;
1116    CurrentFile->IndexType = idxIndexType;
1117    PkgFileName = File;
1118    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1119    Cache.HeaderP->PackageFileCount++;
1120
1121    if (Progress != 0)
1122       Progress->SubProgress(Index.Size());
1123    return true;
1124 }
1125                                                                         /*}}}*/
1126 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1127 // ---------------------------------------------------------------------
1128 /* This is used to create handles to strings. Given the same text it
1129    always returns the same number */
1130 map_stringitem_t pkgCacheGenerator::WriteUniqString(const char *S,
1131                                                  unsigned int Size)
1132 {
1133    /* We use a very small transient hash table here, this speeds up generation
1134       by a fair amount on slower machines */
1135    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
1136    if (Bucket != 0 &&
1137        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
1138       return Bucket->String;
1139
1140    // Search for an insertion point
1141    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
1142    int Res = 1;
1143    map_stringitem_t *Last = &Cache.HeaderP->StringList;
1144    for (; I != Cache.StringItemP; Last = &I->NextItem,
1145         I = Cache.StringItemP + I->NextItem)
1146    {
1147       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
1148       if (Res >= 0)
1149          break;
1150    }
1151
1152    // Match
1153    if (Res == 0)
1154    {
1155       Bucket = I;
1156       return I->String;
1157    }
1158
1159    // Get a structure
1160    void const * const oldMap = Map.Data();
1161    map_pointer_t const Item = AllocateInMap(sizeof(pkgCache::StringItem));
1162    if (Item == 0)
1163       return 0;
1164
1165    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1166    if (unlikely(idxString == 0))
1167       return 0;
1168    if (oldMap != Map.Data()) {
1169       Last += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1170       I += (pkgCache::StringItem const * const) Map.Data() - (pkgCache::StringItem const * const) oldMap;
1171    }
1172    *Last = Item;
1173
1174    // Fill in the structure
1175    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
1176    ItemP->NextItem = I - Cache.StringItemP;
1177    ItemP->String = idxString;
1178
1179    Bucket = ItemP;
1180    return ItemP->String;
1181 }
1182                                                                         /*}}}*/
1183 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1184 // ---------------------------------------------------------------------
1185 /* This just verifies that each file in the list of index files exists,
1186    has matching attributes with the cache and the cache does not have
1187    any extra files. */
1188 static bool CheckValidity(const string &CacheFile,
1189                           pkgSourceList &List,
1190                           FileIterator Start,
1191                           FileIterator End,
1192                           MMap **OutMap = 0)
1193 {
1194    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1195    // No file, certainly invalid
1196    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1197    {
1198       if (Debug == true)
1199          std::clog << "CacheFile doesn't exist" << std::endl;
1200       return false;
1201    }
1202
1203    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1204    {
1205       if (Debug == true)
1206          std::clog << "sources.list is newer than the cache" << std::endl;
1207       return false;
1208    }
1209
1210    // Map it
1211    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1212    SPtr<MMap> Map = new MMap(CacheF,0);
1213    pkgCache Cache(Map);
1214    if (_error->PendingError() == true || Map->Size() == 0)
1215    {
1216       if (Debug == true)
1217          std::clog << "Errors are pending or Map is empty()" << std::endl;
1218       _error->Discard();
1219       return false;
1220    }
1221
1222    /* Now we check every index file, see if it is in the cache,
1223       verify the IMS data and check that it is on the disk too.. */
1224    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1225    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1226    for (; Start != End; ++Start)
1227    {
1228       if (Debug == true)
1229          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
1230       if ((*Start)->HasPackages() == false)
1231       {
1232          if (Debug == true)
1233             std::clog << "Has NO packages" << std::endl;
1234          continue;
1235       }
1236
1237       if ((*Start)->Exists() == false)
1238       {
1239 #if 0 // mvo: we no longer give a message here (Default Sources spec)
1240          _error->WarningE("stat",_("Couldn't stat source package list %s"),
1241                           (*Start)->Describe().c_str());
1242 #endif
1243          if (Debug == true)
1244             std::clog << "file doesn't exist" << std::endl;
1245          continue;
1246       }
1247
1248       // FindInCache is also expected to do an IMS check.
1249       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
1250       if (File.end() == true)
1251       {
1252          if (Debug == true)
1253             std::clog << "FindInCache returned end-Pointer" << std::endl;
1254          return false;
1255       }
1256
1257       Visited[File->ID] = true;
1258       if (Debug == true)
1259          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1260    }
1261
1262    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1263       if (Visited[I] == false)
1264       {
1265          if (Debug == true)
1266             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1267          return false;
1268       }
1269
1270    if (_error->PendingError() == true)
1271    {
1272       if (Debug == true)
1273       {
1274          std::clog << "Validity failed because of pending errors:" << std::endl;
1275          _error->DumpErrors();
1276       }
1277       _error->Discard();
1278       return false;
1279    }
1280
1281    if (OutMap != 0)
1282       *OutMap = Map.UnGuard();
1283    return true;
1284 }
1285                                                                         /*}}}*/
1286 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1287 // ---------------------------------------------------------------------
1288 /* Size is kind of an abstract notion that is only used for the progress
1289    meter */
1290 static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
1291 {
1292    map_filesize_t TotalSize = 0;
1293    for (; Start < End; ++Start)
1294    {
1295       if ((*Start)->HasPackages() == false)
1296          continue;
1297       TotalSize += (*Start)->Size();
1298    }
1299    return TotalSize;
1300 }
1301                                                                         /*}}}*/
1302 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1303 // ---------------------------------------------------------------------
1304 /* */
1305 static bool BuildCache(pkgCacheGenerator &Gen,
1306                        OpProgress *Progress,
1307                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1308                        FileIterator Start, FileIterator End)
1309 {
1310    FileIterator I;
1311    for (I = Start; I != End; ++I)
1312    {
1313       if ((*I)->HasPackages() == false)
1314          continue;
1315
1316       if ((*I)->Exists() == false)
1317          continue;
1318
1319       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1320       {
1321          _error->Warning("Duplicate sources.list entry %s",
1322                          (*I)->Describe().c_str());
1323          continue;
1324       }
1325
1326       map_filesize_t Size = (*I)->Size();
1327       if (Progress != NULL)
1328          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1329       CurrentSize += Size;
1330
1331       if ((*I)->Merge(Gen,Progress) == false)
1332          return false;
1333    }
1334
1335    if (Gen.HasFileDeps() == true)
1336    {
1337       if (Progress != NULL)
1338          Progress->Done();
1339       TotalSize = ComputeSize(Start, End);
1340       CurrentSize = 0;
1341       for (I = Start; I != End; ++I)
1342       {
1343          map_filesize_t Size = (*I)->Size();
1344          if (Progress != NULL)
1345             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1346          CurrentSize += Size;
1347          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1348             return false;
1349       }
1350    }
1351
1352    return true;
1353 }
1354                                                                         /*}}}*/
1355 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1356 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1357    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1358    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1359    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1360    Flags |= MMap::Moveable;
1361    if (_config->FindB("APT::Cache-Fallback", false) == true)
1362       Flags |= MMap::Fallback;
1363    if (CacheF != NULL)
1364       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1365    else
1366       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1367 }
1368                                                                         /*}}}*/
1369 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1370 // ---------------------------------------------------------------------
1371 /* This makes sure that the status cache (the cache that has all
1372    index files from the sources list and all local ones) is ready
1373    to be mmaped. If OutMap is not zero then a MMap object representing
1374    the cache will be stored there. This is pretty much mandetory if you
1375    are using AllowMem. AllowMem lets the function be run as non-root
1376    where it builds the cache 'fast' into a memory buffer. */
1377 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1378                         MMap **OutMap, bool AllowMem)
1379    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1380 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1381                         MMap **OutMap,bool AllowMem)
1382 {
1383    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1384
1385    std::vector<pkgIndexFile *> Files;
1386    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1387         i != List.end();
1388         ++i)
1389    {
1390       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1391       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1392            j != Indexes->end();
1393            ++j)
1394          Files.push_back (*j);
1395    }
1396
1397    map_filesize_t const EndOfSource = Files.size();
1398    if (_system->AddStatusFiles(Files) == false)
1399       return false;
1400
1401    // Decide if we can write to the files..
1402    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1403    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1404
1405    // ensure the cache directory exists
1406    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1407    {
1408       string dir = _config->FindDir("Dir::Cache");
1409       size_t const len = dir.size();
1410       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1411          dir = dir.substr(0, len - 5);
1412       if (CacheFile.empty() == false)
1413          CreateDirectory(dir, flNotFile(CacheFile));
1414       if (SrcCacheFile.empty() == false)
1415          CreateDirectory(dir, flNotFile(SrcCacheFile));
1416    }
1417
1418    // Decide if we can write to the cache
1419    bool Writeable = false;
1420    if (CacheFile.empty() == false)
1421       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1422    else
1423       if (SrcCacheFile.empty() == false)
1424          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1425    if (Debug == true)
1426       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1427
1428    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1429       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1430
1431    if (Progress != NULL)
1432       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1433
1434    // Cache is OK, Fin.
1435    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1436    {
1437       if (Progress != NULL)
1438          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1439       if (Debug == true)
1440          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1441       return true;
1442    }
1443    else if (Debug == true)
1444          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1445
1446    /* At this point we know we need to reconstruct the package cache,
1447       begin. */
1448    SPtr<FileFd> CacheF;
1449    SPtr<DynamicMMap> Map;
1450    if (Writeable == true && CacheFile.empty() == false)
1451    {
1452       _error->PushToStack();
1453       unlink(CacheFile.c_str());
1454       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1455       fchmod(CacheF->Fd(),0644);
1456       Map = CreateDynamicMMap(CacheF, MMap::Public);
1457       if (_error->PendingError() == true)
1458       {
1459          delete CacheF.UnGuard();
1460          delete Map.UnGuard();
1461          if (Debug == true)
1462             std::clog << "Open filebased MMap FAILED" << std::endl;
1463          Writeable = false;
1464          if (AllowMem == false)
1465          {
1466             _error->MergeWithStack();
1467             return false;
1468          }
1469          _error->RevertToStack();
1470       }
1471       else
1472       {
1473          _error->MergeWithStack();
1474          if (Debug == true)
1475             std::clog << "Open filebased MMap" << std::endl;
1476       }
1477    }
1478    if (Writeable == false || CacheFile.empty() == true)
1479    {
1480       // Just build it in memory..
1481       Map = CreateDynamicMMap(NULL);
1482       if (Debug == true)
1483          std::clog << "Open memory Map (not filebased)" << std::endl;
1484    }
1485
1486    // Lets try the source cache.
1487    map_filesize_t CurrentSize = 0;
1488    map_filesize_t TotalSize = 0;
1489    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1490                      Files.begin()+EndOfSource) == true)
1491    {
1492       if (Debug == true)
1493          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1494       // Preload the map with the source cache
1495       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1496       map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
1497       if ((alloc == 0 && _error->PendingError())
1498                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1499                                 SCacheF.Size()) == false)
1500          return false;
1501
1502       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1503
1504       // Build the status cache
1505       pkgCacheGenerator Gen(Map.Get(),Progress);
1506       if (_error->PendingError() == true)
1507          return false;
1508       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1509                      Files.begin()+EndOfSource,Files.end()) == false)
1510          return false;
1511    }
1512    else
1513    {
1514       if (Debug == true)
1515          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1516       TotalSize = ComputeSize(Files.begin(),Files.end());
1517
1518       // Build the source cache
1519       pkgCacheGenerator Gen(Map.Get(),Progress);
1520       if (_error->PendingError() == true)
1521          return false;
1522       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1523                      Files.begin(),Files.begin()+EndOfSource) == false)
1524          return false;
1525
1526       // Write it back
1527       if (Writeable == true && SrcCacheFile.empty() == false)
1528       {
1529          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1530          if (_error->PendingError() == true)
1531             return false;
1532
1533          fchmod(SCacheF.Fd(),0644);
1534
1535          // Write out the main data
1536          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1537             return _error->Error(_("IO Error saving source cache"));
1538          SCacheF.Sync();
1539
1540          // Write out the proper header
1541          Gen.GetCache().HeaderP->Dirty = false;
1542          if (SCacheF.Seek(0) == false ||
1543              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1544             return _error->Error(_("IO Error saving source cache"));
1545          Gen.GetCache().HeaderP->Dirty = true;
1546          SCacheF.Sync();
1547       }
1548
1549       // Build the status cache
1550       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1551                      Files.begin()+EndOfSource,Files.end()) == false)
1552          return false;
1553    }
1554    if (Debug == true)
1555       std::clog << "Caches are ready for shipping" << std::endl;
1556
1557    if (_error->PendingError() == true)
1558       return false;
1559    if (OutMap != 0)
1560    {
1561       if (CacheF != 0)
1562       {
1563          delete Map.UnGuard();
1564          *OutMap = new MMap(*CacheF,0);
1565       }
1566       else
1567       {
1568          *OutMap = Map.UnGuard();
1569       }
1570    }
1571
1572    return true;
1573 }
1574                                                                         /*}}}*/
1575 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1576 // ---------------------------------------------------------------------
1577 /* */
1578 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1579    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1580 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1581 {
1582    std::vector<pkgIndexFile *> Files;
1583    map_filesize_t EndOfSource = Files.size();
1584    if (_system->AddStatusFiles(Files) == false)
1585       return false;
1586
1587    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1588    map_filesize_t CurrentSize = 0;
1589    map_filesize_t TotalSize = 0;
1590
1591    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1592
1593    // Build the status cache
1594    if (Progress != NULL)
1595       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1596    pkgCacheGenerator Gen(Map.Get(),Progress);
1597    if (_error->PendingError() == true)
1598       return false;
1599    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1600                   Files.begin()+EndOfSource,Files.end()) == false)
1601       return false;
1602
1603    if (_error->PendingError() == true)
1604       return false;
1605    *OutMap = Map.UnGuard();
1606
1607    return true;
1608 }
1609                                                                         /*}}}*/
1610 // IsDuplicateDescription                                               /*{{{*/
1611 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1612                             MD5SumValue const &CurMd5, std::string const &CurLang)
1613 {
1614    // Descriptions in the same link-list have all the same md5
1615    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1616       return false;
1617    for (; Desc.end() == false; ++Desc)
1618       if (Desc.LanguageCode() == CurLang)
1619          return true;
1620    return false;
1621 }
1622                                                                         /*}}}*/
1623 // CacheGenerator::FinishCache                                          /*{{{*/
1624 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1625 {
1626    return true;
1627 }
1628                                                                         /*}}}*/