apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/metaindex.h>
  28 #include <apt-pkg/fileutl.h>
  29
  30 #include <vector>
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35
  36 #include <apti18n.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Cache.HeaderP->CacheFileSize = Map.Size();
 102    Map.Sync(0,sizeof(pkgCache::Header));
 103 }
 104                                                                         /*}}}*/
 105 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 106    if (oldMap == newMap)
 107       return;
 108
 109    if (_config->FindB("Debug::pkgCacheGen", false))
 110       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 111
 112    Cache.ReMap(false);
 113
 114    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 115
 116    for (size_t i = 0; i < _count(UniqHash); ++i)
 117       if (UniqHash[i] != 0)
 118          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 119
 120    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 121         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 122       (*i)->ReMap(oldMap, newMap);
 123    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 124         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 125       (*i)->ReMap(oldMap, newMap);
 126    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 127         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 128       (*i)->ReMap(oldMap, newMap);
 129    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 130         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 131       (*i)->ReMap(oldMap, newMap);
 132    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 133         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 134       (*i)->ReMap(oldMap, newMap);
 135    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 136         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 137       (*i)->ReMap(oldMap, newMap);
 138    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 139         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 140       (*i)->ReMap(oldMap, newMap);
 141 }                                                                       /*}}}*/
 142 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 143 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 144                                         const unsigned long &Len) {
 145    void const * const oldMap = Map.Data();
 146    map_ptrloc const index = Map.WriteString(String, Len);
 147    if (index != 0)
 148       ReMap(oldMap, Map.Data());
 149    return index;
 150 }
 151                                                                         /*}}}*/
 152 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 153 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 154    void const * const oldMap = Map.Data();
 155    map_ptrloc const index = Map.WriteString(String);
 156    if (index != 0)
 157       ReMap(oldMap, Map.Data());
 158    return index;
 159 }
 160                                                                         /*}}}*/
 161 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 162    void const * const oldMap = Map.Data();
 163    map_ptrloc const index = Map.Allocate(size);
 164    if (index != 0)
 165       ReMap(oldMap, Map.Data());
 166    return index;
 167 }
 168                                                                         /*}}}*/
 169 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 170 // ---------------------------------------------------------------------
 171 /* This provides the generation of the entries in the cache. Each loop
 172    goes through a single package record from the underlying parse engine. */
 173 bool pkgCacheGenerator::MergeList(ListParser &List,
 174                                   pkgCache::VerIterator *OutVer)
 175 {
 176    List.Owner = this;
 177
 178    unsigned int Counter = 0;
 179    while (List.Step() == true)
 180    {
 181       string const PackageName = List.Package();
 182       if (PackageName.empty() == true)
 183          return false;
 184
 185       Counter++;
 186       if (Counter % 100 == 0 && Progress != 0)
 187          Progress->Progress(List.Offset());
 188
 189       string Arch = List.Architecture();
 190       string const Version = List.Version();
 191       if (Version.empty() == true && Arch.empty() == true)
 192       {
 193          if (MergeListGroup(List, PackageName) == false)
 194             return false;
 195       }
 196
 197       if (Arch.empty() == true)
 198          Arch = _config->Find("APT::Architecture");
 199
 200       // Get a pointer to the package structure
 201       pkgCache::PkgIterator Pkg;
 202       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 203       if (NewPackage(Pkg, PackageName, Arch) == false)
 204          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 205
 206
 207       if (Version.empty() == true)
 208       {
 209          if (MergeListPackage(List, Pkg) == false)
 210             return false;
 211       }
 212       else
 213       {
 214          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 215             return false;
 216       }
 217
 218       if (OutVer != 0)
 219       {
 220          FoundFileDeps |= List.HasFileDeps();
 221          return true;
 222       }
 223    }
 224
 225    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 226       return _error->Error(_("Wow, you exceeded the number of package "
 227                              "names this APT is capable of."));
 228    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 229       return _error->Error(_("Wow, you exceeded the number of versions "
 230                              "this APT is capable of."));
 231    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 232       return _error->Error(_("Wow, you exceeded the number of descriptions "
 233                              "this APT is capable of."));
 234    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 235       return _error->Error(_("Wow, you exceeded the number of dependencies "
 236                              "this APT is capable of."));
 237
 238    FoundFileDeps |= List.HasFileDeps();
 239    return true;
 240 }
 241 // CacheGenerator::MergeListGroup                                       /*{{{*/
 242 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 243 {
 244    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 245    if (Grp.end() == true)
 246       return _error->Error("Information merge for non-existent group %s requested", GrpName.c_str());
 247    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 248
 249    pkgCache::PkgIterator Pkg;
 250    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 251    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 252       if (MergeListPackage(List, Pkg) == false)
 253          return false;
 254
 255    return true;
 256 }
 257                                                                         /*}}}*/
 258 // CacheGenerator::MergeListPackage                                     /*{{{*/
 259 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 260 {
 261    // we first process the package, then the descriptions
 262    // (for deb this package processing is in fact a no-op)
 263    pkgCache::VerIterator Ver(Cache);
 264    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 265    if (List.UsePackage(Pkg, Ver) == false)
 266       return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 267                            Pkg.Name());
 268
 269    // Find the right version to write the description
 270    MD5SumValue CurMd5 = List.Description_md5();
 271
 272    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 273    {
 274       pkgCache::DescIterator Desc = Ver.DescriptionList();
 275       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 276       map_ptrloc *LastDesc = &Ver->DescriptionList;
 277
 278
 279       // don't add a new description if we have one for the given
 280       // md5 && language
 281       bool duplicate = false;
 282       for ( ; Desc.end() == false; ++Desc)
 283          if (MD5SumValue(Desc.md5()) == CurMd5 &&
 284              Desc.LanguageCode() == List.DescriptionLanguage())
 285                duplicate=true;
 286       if (duplicate == true)
 287          continue;
 288
 289       for (Desc = Ver.DescriptionList();
 290            Desc.end() == false;
 291             LastDesc = &Desc->NextDesc, ++Desc)
 292       {
 293          if (MD5SumValue(Desc.md5()) != CurMd5)
 294             continue;
 295
 296          // Add new description
 297          void const * const oldMap = Map.Data();
 298          map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 299          if (oldMap != Map.Data())
 300             LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 301          *LastDesc = descindex;
 302          Desc->ParentPkg = Pkg.Index();
 303
 304          if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 305             return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"), Pkg.Name());
 306          break;
 307        }
 308    }
 309
 310    return true;
 311 }
 312                                                                         /*}}}*/
 313 // CacheGenerator::MergeListVersion                                     /*{{{*/
 314 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 315                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 316 {
 317    pkgCache::VerIterator Ver = Pkg.VersionList();
 318    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 319    map_ptrloc *LastVer = &Pkg->VersionList;
 320    void const * oldMap = Map.Data();
 321
 322    unsigned long const Hash = List.VersionHash();
 323    if (Ver.end() == false)
 324    {
 325       /* We know the list is sorted so we use that fact in the search.
 326          Insertion of new versions is done with correct sorting */
 327       int Res = 1;
 328       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 329       {
 330          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 331          // Version is higher as current version - insert here
 332          if (Res > 0)
 333             break;
 334          // Versionstrings are equal - is hash also equal?
 335          if (Res == 0 && Ver->Hash == Hash)
 336             break;
 337          // proceed with the next till we have either the right
 338          // or we found another version (which will be lower)
 339       }
 340
 341       /* We already have a version for this item, record that we saw it */
 342       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 343       {
 344          if (List.UsePackage(Pkg,Ver) == false)
 345             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 346                                  Pkg.Name());
 347
 348          if (NewFileVer(Ver,List) == false)
 349             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 350                                  Pkg.Name());
 351
 352          // Read only a single record and return
 353          if (OutVer != 0)
 354          {
 355             *OutVer = Ver;
 356             return true;
 357          }
 358
 359          return true;
 360       }
 361    }
 362
 363    // Add a new version
 364    map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 365    if (verindex == 0 && _error->PendingError())
 366       return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 367                            Pkg.Name(), 1);
 368
 369    if (oldMap != Map.Data())
 370          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 371    *LastVer = verindex;
 372    Ver->ParentPkg = Pkg.Index();
 373    Ver->Hash = Hash;
 374
 375    if (List.NewVersion(Ver) == false)
 376       return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 377                            Pkg.Name(), 2);
 378
 379    if (List.UsePackage(Pkg,Ver) == false)
 380       return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 381                            Pkg.Name());
 382
 383    if (NewFileVer(Ver,List) == false)
 384       return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 385                            Pkg.Name(), 3);
 386
 387    // Read only a single record and return
 388    if (OutVer != 0)
 389    {
 390       *OutVer = Ver;
 391       return true;
 392    }
 393
 394    /* Record the Description data. Description data always exist in
 395       Packages and Translation-* files. */
 396    pkgCache::DescIterator Desc = Ver.DescriptionList();
 397    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 398    map_ptrloc *LastDesc = &Ver->DescriptionList;
 399
 400    // Skip to the end of description set
 401    for (; Desc.end() == false; LastDesc = &Desc->NextDesc, ++Desc);
 402
 403    // Add new description
 404    oldMap = Map.Data();
 405    map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 406    if (oldMap != Map.Data())
 407        LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 408    *LastDesc = descindex;
 409    Desc->ParentPkg = Pkg.Index();
 410
 411    if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 412       return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),Pkg.Name());
 413
 414    return true;
 415 }
 416                                                                         /*}}}*/
 417                                                                         /*}}}*/
 418 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 419 // ---------------------------------------------------------------------
 420 /* If we found any file depends while parsing the main list we need to
 421    resolve them. Since it is undesired to load the entire list of files
 422    into the cache as virtual packages we do a two stage effort. MergeList
 423    identifies the file depends and this creates Provdies for them by
 424    re-parsing all the indexs. */
 425 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 426 {
 427    List.Owner = this;
 428
 429    unsigned int Counter = 0;
 430    while (List.Step() == true)
 431    {
 432       string PackageName = List.Package();
 433       if (PackageName.empty() == true)
 434          return false;
 435       string Version = List.Version();
 436       if (Version.empty() == true)
 437          continue;
 438
 439       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 440       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 441       if (Pkg.end() == true)
 442          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 443                                 PackageName.c_str());
 444       Counter++;
 445       if (Counter % 100 == 0 && Progress != 0)
 446          Progress->Progress(List.Offset());
 447
 448       unsigned long Hash = List.VersionHash();
 449       pkgCache::VerIterator Ver = Pkg.VersionList();
 450       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 451       for (; Ver.end() == false; ++Ver)
 452       {
 453          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 454          {
 455             if (List.CollectFileProvides(Cache,Ver) == false)
 456                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 457             break;
 458          }
 459       }
 460
 461       if (Ver.end() == true)
 462          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 463    }
 464
 465    return true;
 466 }
 467                                                                         /*}}}*/
 468 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 469 // ---------------------------------------------------------------------
 470 /* This creates a new group structure and adds it to the hash table */
 471 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 472 {
 473    Grp = Cache.FindGrp(Name);
 474    if (Grp.end() == false)
 475       return true;
 476
 477    // Get a structure
 478    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 479    if (unlikely(Group == 0))
 480       return false;
 481
 482    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 483    map_ptrloc const idxName = WriteStringInMap(Name);
 484    if (unlikely(idxName == 0))
 485       return false;
 486    Grp->Name = idxName;
 487
 488    // Insert it into the hash table
 489    unsigned long const Hash = Cache.Hash(Name);
 490    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 491    Cache.HeaderP->GrpHashTable[Hash] = Group;
 492
 493    Grp->ID = Cache.HeaderP->GroupCount++;
 494    return true;
 495 }
 496                                                                         /*}}}*/
 497 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 498 // ---------------------------------------------------------------------
 499 /* This creates a new package structure and adds it to the hash table */
 500 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 501                                         const string &Arch) {
 502    pkgCache::GrpIterator Grp;
 503    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 504    if (unlikely(NewGroup(Grp, Name) == false))
 505       return false;
 506
 507    Pkg = Grp.FindPkg(Arch);
 508       if (Pkg.end() == false)
 509          return true;
 510
 511    // Get a structure
 512    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 513    if (unlikely(Package == 0))
 514       return false;
 515    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 516
 517    // Insert the package into our package list
 518    if (Grp->FirstPackage == 0) // the group is new
 519    {
 520       // Insert it into the hash table
 521       unsigned long const Hash = Cache.Hash(Name);
 522       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 523       Cache.HeaderP->PkgHashTable[Hash] = Package;
 524       Grp->FirstPackage = Package;
 525    }
 526    else // Group the Packages together
 527    {
 528       // this package is the new last package
 529       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 530       Pkg->NextPackage = LastPkg->NextPackage;
 531       LastPkg->NextPackage = Package;
 532    }
 533    Grp->LastPackage = Package;
 534
 535    // Set the name, arch and the ID
 536    Pkg->Name = Grp->Name;
 537    Pkg->Group = Grp.Index();
 538    // all is mapped to the native architecture
 539    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 540    if (unlikely(idxArch == 0))
 541       return false;
 542    Pkg->Arch = idxArch;
 543    Pkg->ID = Cache.HeaderP->PackageCount++;
 544
 545    return true;
 546 }
 547                                                                         /*}}}*/
 548 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* */
 551 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 552                                    ListParser &List)
 553 {
 554    if (CurrentFile == 0)
 555       return true;
 556
 557    // Get a structure
 558    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 559    if (VerFile == 0)
 560       return 0;
 561
 562    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 563    VF->File = CurrentFile - Cache.PkgFileP;
 564
 565    // Link it to the end of the list
 566    map_ptrloc *Last = &Ver->FileList;
 567    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 568       Last = &V->NextFile;
 569    VF->NextFile = *Last;
 570    *Last = VF.Index();
 571
 572    VF->Offset = List.Offset();
 573    VF->Size = List.Size();
 574    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 575       Cache.HeaderP->MaxVerFileSize = VF->Size;
 576    Cache.HeaderP->VerFileCount++;
 577
 578    return true;
 579 }
 580                                                                         /*}}}*/
 581 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* This puts a version structure in the linked list */
 584 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 585                                             const string &VerStr,
 586                                             unsigned long Next)
 587 {
 588    // Get a structure
 589    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 590    if (Version == 0)
 591       return 0;
 592
 593    // Fill it in
 594    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 595    Ver->NextVer = Next;
 596    Ver->ID = Cache.HeaderP->VersionCount++;
 597    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 598    if (unlikely(idxVerStr == 0))
 599       return 0;
 600    Ver->VerStr = idxVerStr;
 601
 602    return Version;
 603 }
 604                                                                         /*}}}*/
 605 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 606 // ---------------------------------------------------------------------
 607 /* */
 608 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 609                                    ListParser &List)
 610 {
 611    if (CurrentFile == 0)
 612       return true;
 613
 614    // Get a structure
 615    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 616    if (DescFile == 0)
 617       return false;
 618
 619    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 620    DF->File = CurrentFile - Cache.PkgFileP;
 621
 622    // Link it to the end of the list
 623    map_ptrloc *Last = &Desc->FileList;
 624    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 625       Last = &D->NextFile;
 626
 627    DF->NextFile = *Last;
 628    *Last = DF.Index();
 629
 630    DF->Offset = List.Offset();
 631    DF->Size = List.Size();
 632    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 633       Cache.HeaderP->MaxDescFileSize = DF->Size;
 634    Cache.HeaderP->DescFileCount++;
 635
 636    return true;
 637 }
 638                                                                         /*}}}*/
 639 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 640 // ---------------------------------------------------------------------
 641 /* This puts a description structure in the linked list */
 642 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 643                                             const string &Lang,
 644                                             const MD5SumValue &md5sum,
 645                                             map_ptrloc Next)
 646 {
 647    // Get a structure
 648    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 649    if (Description == 0)
 650       return 0;
 651
 652    // Fill it in
 653    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 654    Desc->NextDesc = Next;
 655    Desc->ID = Cache.HeaderP->DescriptionCount++;
 656    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 657    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 658    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 659       return 0;
 660    Desc->language_code = idxlanguage_code;
 661    Desc->md5sum = idxmd5sum;
 662
 663    return Description;
 664 }
 665                                                                         /*}}}*/
 666 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 667 // ---------------------------------------------------------------------
 668 /* This prepares the Cache for delivery */
 669 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 670 {
 671    // FIXME: add progress reporting for this operation
 672    // Do we have different architectures in your groups ?
 673    vector<string> archs = APT::Configuration::getArchitectures();
 674    if (archs.size() > 1)
 675    {
 676       // Create Conflicts in between the group
 677       pkgCache::GrpIterator G = GetCache().GrpBegin();
 678       Dynamic<pkgCache::GrpIterator> DynG(G);
 679       for (; G.end() != true; ++G)
 680       {
 681          string const PkgName = G.Name();
 682          pkgCache::PkgIterator P = G.PackageList();
 683          Dynamic<pkgCache::PkgIterator> DynP(P);
 684          for (; P.end() != true; P = G.NextPkg(P))
 685          {
 686             pkgCache::PkgIterator allPkg;
 687             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 688             pkgCache::VerIterator V = P.VersionList();
 689             Dynamic<pkgCache::VerIterator> DynV(V);
 690             for (; V.end() != true; ++V)
 691             {
 692                // copy P.Arch() into a string here as a cache remap
 693                // in NewDepends() later may alter the pointer location
 694                string Arch = P.Arch() == NULL ? "" : P.Arch();
 695                map_ptrloc *OldDepLast = NULL;
 696                /* MultiArch handling introduces a lot of implicit Dependencies:
 697                 - MultiArch: same → Co-Installable if they have the same version
 698                 - Architecture: all → Need to be Co-Installable for internal reasons
 699                 - All others conflict with all other group members */
 700                bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 701                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 702                {
 703                   if (*A == Arch)
 704                      continue;
 705                   /* We allow only one installed arch at the time
 706                      per group, therefore each group member conflicts
 707                      with all other group members */
 708                   pkgCache::PkgIterator D = G.FindPkg(*A);
 709                   Dynamic<pkgCache::PkgIterator> DynD(D);
 710                   if (D.end() == true)
 711                      continue;
 712                   if (coInstall == true)
 713                   {
 714                      // Replaces: ${self}:other ( << ${binary:Version})
 715                      NewDepends(D, V, V.VerStr(),
 716                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 717                                 OldDepLast);
 718                      // Breaks: ${self}:other (!= ${binary:Version})
 719                      NewDepends(D, V, V.VerStr(),
 720                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 721                                 OldDepLast);
 722                   } else {
 723                         // Conflicts: ${self}:other
 724                         NewDepends(D, V, "",
 725                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 726                                    OldDepLast);
 727                   }
 728                }
 729             }
 730          }
 731       }
 732    }
 733    return true;
 734 }
 735                                                                         /*}}}*/
 736 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 737 // ---------------------------------------------------------------------
 738 /* This creates a dependency element in the tree. It is linked to the
 739    version and to the package that it is pointing to. */
 740 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 741                                    pkgCache::VerIterator &Ver,
 742                                    string const &Version,
 743                                    unsigned int const &Op,
 744                                    unsigned int const &Type,
 745                                    map_ptrloc* &OldDepLast)
 746 {
 747    void const * const oldMap = Map.Data();
 748    // Get a structure
 749    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 750    if (unlikely(Dependency == 0))
 751       return false;
 752
 753    // Fill it in
 754    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 755    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 756    Dep->ParentVer = Ver.Index();
 757    Dep->Type = Type;
 758    Dep->CompareOp = Op;
 759    Dep->ID = Cache.HeaderP->DependsCount++;
 760
 761    // Probe the reverse dependency list for a version string that matches
 762    if (Version.empty() == false)
 763    {
 764 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 765          if (I->Version != 0 && I.TargetVer() == Version)
 766             Dep->Version = I->Version;*/
 767       if (Dep->Version == 0) {
 768          map_ptrloc const index = WriteStringInMap(Version);
 769          if (unlikely(index == 0))
 770             return false;
 771          Dep->Version = index;
 772       }
 773    }
 774
 775    // Link it to the package
 776    Dep->Package = Pkg.Index();
 777    Dep->NextRevDepends = Pkg->RevDepends;
 778    Pkg->RevDepends = Dep.Index();
 779
 780    // Do we know where to link the Dependency to?
 781    if (OldDepLast == NULL)
 782    {
 783       OldDepLast = &Ver->DependsList;
 784       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 785          OldDepLast = &D->NextDepends;
 786    } else if (oldMap != Map.Data())
 787       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 788
 789    Dep->NextDepends = *OldDepLast;
 790    *OldDepLast = Dep.Index();
 791    OldDepLast = &Dep->NextDepends;
 792
 793    return true;
 794 }
 795                                                                         /*}}}*/
 796 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 797 // ---------------------------------------------------------------------
 798 /* This creates a Group and the Package to link this dependency to if
 799    needed and handles also the caching of the old endpoint */
 800 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 801                                                const string &PackageName,
 802                                                const string &Arch,
 803                                                const string &Version,
 804                                                unsigned int Op,
 805                                                unsigned int Type)
 806 {
 807    pkgCache::GrpIterator Grp;
 808    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 809    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 810       return false;
 811
 812    // Locate the target package
 813    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 814    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 815    if (Pkg.end() == true) {
 816       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 817          return false;
 818    }
 819
 820    // Is it a file dependency?
 821    if (unlikely(PackageName[0] == '/'))
 822       FoundFileDeps = true;
 823
 824    /* Caching the old end point speeds up generation substantially */
 825    if (OldDepVer != Ver) {
 826       OldDepLast = NULL;
 827       OldDepVer = Ver;
 828    }
 829
 830    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 831 }
 832                                                                         /*}}}*/
 833 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 834 // ---------------------------------------------------------------------
 835 /* */
 836 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 837                                                 const string &PkgName,
 838                                                 const string &PkgArch,
 839                                                 const string &Version)
 840 {
 841    pkgCache &Cache = Owner->Cache;
 842
 843    // We do not add self referencing provides
 844    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 845         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 846       return true;
 847
 848    // Get a structure
 849    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 850    if (unlikely(Provides == 0))
 851       return false;
 852    Cache.HeaderP->ProvidesCount++;
 853
 854    // Fill it in
 855    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 856    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 857    Prv->Version = Ver.Index();
 858    Prv->NextPkgProv = Ver->ProvidesList;
 859    Ver->ProvidesList = Prv.Index();
 860    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 861       return false;
 862
 863    // Locate the target package
 864    pkgCache::PkgIterator Pkg;
 865    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 866    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 867       return false;
 868
 869    // Link it to the package
 870    Prv->ParentPkg = Pkg.Index();
 871    Prv->NextProvides = Pkg->ProvidesList;
 872    Pkg->ProvidesList = Prv.Index();
 873
 874    return true;
 875 }
 876                                                                         /*}}}*/
 877 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 878 // ---------------------------------------------------------------------
 879 /* This is used to select which file is to be associated with all newly
 880    added versions. The caller is responsible for setting the IMS fields. */
 881 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 882                                    const pkgIndexFile &Index,
 883                                    unsigned long Flags)
 884 {
 885    // Get some space for the structure
 886    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 887    if (unlikely(idxFile == 0))
 888       return false;
 889    CurrentFile = Cache.PkgFileP + idxFile;
 890
 891    // Fill it in
 892    map_ptrloc const idxFileName = WriteStringInMap(File);
 893    map_ptrloc const idxSite = WriteUniqString(Site);
 894    if (unlikely(idxFileName == 0 || idxSite == 0))
 895       return false;
 896    CurrentFile->FileName = idxFileName;
 897    CurrentFile->Site = idxSite;
 898    CurrentFile->NextFile = Cache.HeaderP->FileList;
 899    CurrentFile->Flags = Flags;
 900    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 901    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 902    if (unlikely(idxIndexType == 0))
 903       return false;
 904    CurrentFile->IndexType = idxIndexType;
 905    PkgFileName = File;
 906    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 907    Cache.HeaderP->PackageFileCount++;
 908
 909    if (Progress != 0)
 910       Progress->SubProgress(Index.Size());
 911    return true;
 912 }
 913                                                                         /*}}}*/
 914 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 915 // ---------------------------------------------------------------------
 916 /* This is used to create handles to strings. Given the same text it
 917    always returns the same number */
 918 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 919                                                  unsigned int Size)
 920 {
 921    /* We use a very small transient hash table here, this speeds up generation
 922       by a fair amount on slower machines */
 923    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 924    if (Bucket != 0 &&
 925        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 926       return Bucket->String;
 927
 928    // Search for an insertion point
 929    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 930    int Res = 1;
 931    map_ptrloc *Last = &Cache.HeaderP->StringList;
 932    for (; I != Cache.StringItemP; Last = &I->NextItem,
 933         I = Cache.StringItemP + I->NextItem)
 934    {
 935       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 936       if (Res >= 0)
 937          break;
 938    }
 939
 940    // Match
 941    if (Res == 0)
 942    {
 943       Bucket = I;
 944       return I->String;
 945    }
 946
 947    // Get a structure
 948    void const * const oldMap = Map.Data();
 949    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 950    if (Item == 0)
 951       return 0;
 952
 953    map_ptrloc const idxString = WriteStringInMap(S,Size);
 954    if (unlikely(idxString == 0))
 955       return 0;
 956    if (oldMap != Map.Data()) {
 957       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 958       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 959    }
 960    *Last = Item;
 961
 962    // Fill in the structure
 963    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 964    ItemP->NextItem = I - Cache.StringItemP;
 965    ItemP->String = idxString;
 966
 967    Bucket = ItemP;
 968    return ItemP->String;
 969 }
 970                                                                         /*}}}*/
 971 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 972 // ---------------------------------------------------------------------
 973 /* This just verifies that each file in the list of index files exists,
 974    has matching attributes with the cache and the cache does not have
 975    any extra files. */
 976 static bool CheckValidity(const string &CacheFile,
 977                           pkgSourceList &List,
 978                           FileIterator Start,
 979                           FileIterator End,
 980                           MMap **OutMap = 0)
 981 {
 982    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 983    // No file, certainly invalid
 984    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 985    {
 986       if (Debug == true)
 987          std::clog << "CacheFile doesn't exist" << std::endl;
 988       return false;
 989    }
 990
 991    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
 992    {
 993       if (Debug == true)
 994          std::clog << "sources.list is newer than the cache" << std::endl;
 995       return false;
 996    }
 997
 998    // Map it
 999    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1000    SPtr<MMap> Map = new MMap(CacheF,0);
1001    pkgCache Cache(Map);
1002    if (_error->PendingError() == true || Map->Size() == 0)
1003    {
1004       if (Debug == true)
1005          std::clog << "Errors are pending or Map is empty()" << std::endl;
1006       _error->Discard();
1007       return false;
1008    }
1009
1010    /* Now we check every index file, see if it is in the cache,
1011       verify the IMS data and check that it is on the disk too.. */
1012    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1013    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1014    for (; Start != End; ++Start)
1015    {
1016       if (Debug == true)
1017          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
1018       if ((*Start)->HasPackages() == false)
1019       {
1020          if (Debug == true)
1021             std::clog << "Has NO packages" << std::endl;
1022          continue;
1023       }
1024
1025       if ((*Start)->Exists() == false)
1026       {
1027 #if 0 // mvo: we no longer give a message here (Default Sources spec)
1028          _error->WarningE("stat",_("Couldn't stat source package list %s"),
1029                           (*Start)->Describe().c_str());
1030 #endif
1031          if (Debug == true)
1032             std::clog << "file doesn't exist" << std::endl;
1033          continue;
1034       }
1035
1036       // FindInCache is also expected to do an IMS check.
1037       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
1038       if (File.end() == true)
1039       {
1040          if (Debug == true)
1041             std::clog << "FindInCache returned end-Pointer" << std::endl;
1042          return false;
1043       }
1044
1045       Visited[File->ID] = true;
1046       if (Debug == true)
1047          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1048    }
1049
1050    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1051       if (Visited[I] == false)
1052       {
1053          if (Debug == true)
1054             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1055          return false;
1056       }
1057
1058    if (_error->PendingError() == true)
1059    {
1060       if (Debug == true)
1061       {
1062          std::clog << "Validity failed because of pending errors:" << std::endl;
1063          _error->DumpErrors();
1064       }
1065       _error->Discard();
1066       return false;
1067    }
1068
1069    if (OutMap != 0)
1070       *OutMap = Map.UnGuard();
1071    return true;
1072 }
1073                                                                         /*}}}*/
1074 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1075 // ---------------------------------------------------------------------
1076 /* Size is kind of an abstract notion that is only used for the progress
1077    meter */
1078 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1079 {
1080    unsigned long TotalSize = 0;
1081    for (; Start != End; ++Start)
1082    {
1083       if ((*Start)->HasPackages() == false)
1084          continue;
1085       TotalSize += (*Start)->Size();
1086    }
1087    return TotalSize;
1088 }
1089                                                                         /*}}}*/
1090 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1091 // ---------------------------------------------------------------------
1092 /* */
1093 static bool BuildCache(pkgCacheGenerator &Gen,
1094                        OpProgress *Progress,
1095                        unsigned long &CurrentSize,unsigned long TotalSize,
1096                        FileIterator Start, FileIterator End)
1097 {
1098    FileIterator I;
1099    for (I = Start; I != End; ++I)
1100    {
1101       if ((*I)->HasPackages() == false)
1102          continue;
1103
1104       if ((*I)->Exists() == false)
1105          continue;
1106
1107       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1108       {
1109          _error->Warning("Duplicate sources.list entry %s",
1110                          (*I)->Describe().c_str());
1111          continue;
1112       }
1113
1114       unsigned long Size = (*I)->Size();
1115       if (Progress != NULL)
1116          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1117       CurrentSize += Size;
1118
1119       if ((*I)->Merge(Gen,Progress) == false)
1120          return false;
1121    }
1122
1123    if (Gen.HasFileDeps() == true)
1124    {
1125       if (Progress != NULL)
1126          Progress->Done();
1127       TotalSize = ComputeSize(Start, End);
1128       CurrentSize = 0;
1129       for (I = Start; I != End; ++I)
1130       {
1131          unsigned long Size = (*I)->Size();
1132          if (Progress != NULL)
1133             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1134          CurrentSize += Size;
1135          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1136             return false;
1137       }
1138    }
1139
1140    return true;
1141 }
1142                                                                         /*}}}*/
1143 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1144 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1145    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1146    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1147    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1148    Flags |= MMap::Moveable;
1149    if (_config->FindB("APT::Cache-Fallback", false) == true)
1150       Flags |= MMap::Fallback;
1151    if (CacheF != NULL)
1152       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1153    else
1154       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1155 }
1156                                                                         /*}}}*/
1157 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1158 // ---------------------------------------------------------------------
1159 /* This makes sure that the status cache (the cache that has all
1160    index files from the sources list and all local ones) is ready
1161    to be mmaped. If OutMap is not zero then a MMap object representing
1162    the cache will be stored there. This is pretty much mandetory if you
1163    are using AllowMem. AllowMem lets the function be run as non-root
1164    where it builds the cache 'fast' into a memory buffer. */
1165 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1166                         MMap **OutMap, bool AllowMem)
1167    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1168 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1169                         MMap **OutMap,bool AllowMem)
1170 {
1171    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1172
1173    vector<pkgIndexFile *> Files;
1174    for (vector<metaIndex *>::const_iterator i = List.begin();
1175         i != List.end();
1176         ++i)
1177    {
1178       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1179       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1180            j != Indexes->end();
1181            ++j)
1182          Files.push_back (*j);
1183    }
1184
1185    unsigned long const EndOfSource = Files.size();
1186    if (_system->AddStatusFiles(Files) == false)
1187       return false;
1188
1189    // Decide if we can write to the files..
1190    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1191    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1192
1193    // ensure the cache directory exists
1194    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1195    {
1196       string dir = _config->FindDir("Dir::Cache");
1197       size_t const len = dir.size();
1198       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1199          dir = dir.substr(0, len - 5);
1200       if (CacheFile.empty() == false)
1201          CreateDirectory(dir, flNotFile(CacheFile));
1202       if (SrcCacheFile.empty() == false)
1203          CreateDirectory(dir, flNotFile(SrcCacheFile));
1204    }
1205
1206    // Decide if we can write to the cache
1207    bool Writeable = false;
1208    if (CacheFile.empty() == false)
1209       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1210    else
1211       if (SrcCacheFile.empty() == false)
1212          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1213    if (Debug == true)
1214       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1215
1216    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1217       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1218
1219    if (Progress != NULL)
1220       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1221
1222    // Cache is OK, Fin.
1223    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1224    {
1225       if (Progress != NULL)
1226          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1227       if (Debug == true)
1228          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1229       return true;
1230    }
1231    else if (Debug == true)
1232          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1233
1234    /* At this point we know we need to reconstruct the package cache,
1235       begin. */
1236    SPtr<FileFd> CacheF;
1237    SPtr<DynamicMMap> Map;
1238    if (Writeable == true && CacheFile.empty() == false)
1239    {
1240       _error->PushToStack();
1241       unlink(CacheFile.c_str());
1242       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1243       fchmod(CacheF->Fd(),0644);
1244       Map = CreateDynamicMMap(CacheF, MMap::Public);
1245       if (_error->PendingError() == true)
1246       {
1247          delete CacheF.UnGuard();
1248          delete Map.UnGuard();
1249          if (Debug == true)
1250             std::clog << "Open filebased MMap FAILED" << std::endl;
1251          Writeable = false;
1252          if (AllowMem == false)
1253          {
1254             _error->MergeWithStack();
1255             return false;
1256          }
1257          _error->RevertToStack();
1258       }
1259       else if (Debug == true)
1260       {
1261          _error->MergeWithStack();
1262          std::clog << "Open filebased MMap" << std::endl;
1263       }
1264    }
1265    if (Writeable == false || CacheFile.empty() == true)
1266    {
1267       // Just build it in memory..
1268       Map = CreateDynamicMMap(NULL);
1269       if (Debug == true)
1270          std::clog << "Open memory Map (not filebased)" << std::endl;
1271    }
1272
1273    // Lets try the source cache.
1274    unsigned long CurrentSize = 0;
1275    unsigned long TotalSize = 0;
1276    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1277                      Files.begin()+EndOfSource) == true)
1278    {
1279       if (Debug == true)
1280          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1281       // Preload the map with the source cache
1282       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1283       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1284       if ((alloc == 0 && _error->PendingError())
1285                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1286                                 SCacheF.Size()) == false)
1287          return false;
1288
1289       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1290
1291       // Build the status cache
1292       pkgCacheGenerator Gen(Map.Get(),Progress);
1293       if (_error->PendingError() == true)
1294          return false;
1295       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1296                      Files.begin()+EndOfSource,Files.end()) == false)
1297          return false;
1298
1299       // FIXME: move me to a better place
1300       Gen.FinishCache(Progress);
1301    }
1302    else
1303    {
1304       if (Debug == true)
1305          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1306       TotalSize = ComputeSize(Files.begin(),Files.end());
1307
1308       // Build the source cache
1309       pkgCacheGenerator Gen(Map.Get(),Progress);
1310       if (_error->PendingError() == true)
1311          return false;
1312       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1313                      Files.begin(),Files.begin()+EndOfSource) == false)
1314          return false;
1315
1316       // Write it back
1317       if (Writeable == true && SrcCacheFile.empty() == false)
1318       {
1319          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1320          if (_error->PendingError() == true)
1321             return false;
1322
1323          fchmod(SCacheF.Fd(),0644);
1324
1325          // Write out the main data
1326          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1327             return _error->Error(_("IO Error saving source cache"));
1328          SCacheF.Sync();
1329
1330          // Write out the proper header
1331          Gen.GetCache().HeaderP->Dirty = false;
1332          if (SCacheF.Seek(0) == false ||
1333              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1334             return _error->Error(_("IO Error saving source cache"));
1335          Gen.GetCache().HeaderP->Dirty = true;
1336          SCacheF.Sync();
1337       }
1338
1339       // Build the status cache
1340       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1341                      Files.begin()+EndOfSource,Files.end()) == false)
1342          return false;
1343
1344       // FIXME: move me to a better place
1345       Gen.FinishCache(Progress);
1346    }
1347    if (Debug == true)
1348       std::clog << "Caches are ready for shipping" << std::endl;
1349
1350    if (_error->PendingError() == true)
1351       return false;
1352    if (OutMap != 0)
1353    {
1354       if (CacheF != 0)
1355       {
1356          delete Map.UnGuard();
1357          *OutMap = new MMap(*CacheF,0);
1358       }
1359       else
1360       {
1361          *OutMap = Map.UnGuard();
1362       }
1363    }
1364
1365    return true;
1366 }
1367                                                                         /*}}}*/
1368 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1369 // ---------------------------------------------------------------------
1370 /* */
1371 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1372    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1373 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1374 {
1375    vector<pkgIndexFile *> Files;
1376    unsigned long EndOfSource = Files.size();
1377    if (_system->AddStatusFiles(Files) == false)
1378       return false;
1379
1380    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1381    unsigned long CurrentSize = 0;
1382    unsigned long TotalSize = 0;
1383
1384    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1385
1386    // Build the status cache
1387    if (Progress != NULL)
1388       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1389    pkgCacheGenerator Gen(Map.Get(),Progress);
1390    if (_error->PendingError() == true)
1391       return false;
1392    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1393                   Files.begin()+EndOfSource,Files.end()) == false)
1394       return false;
1395
1396    // FIXME: move me to a better place
1397    Gen.FinishCache(Progress);
1398
1399    if (_error->PendingError() == true)
1400       return false;
1401    *OutMap = Map.UnGuard();
1402
1403    return true;
1404 }
1405                                                                         /*}}}*/