apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26
  27 #include <apt-pkg/tagfile.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Map.Sync(0,sizeof(pkgCache::Header));
 102 }
 103                                                                         /*}}}*/
 104 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 105    if (oldMap == newMap)
 106       return;
 107
 108    if (_config->FindB("Debug::pkgCacheGen", false))
 109       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 110
 111    Cache.ReMap(false);
 112
 113    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 114
 115    for (size_t i = 0; i < _count(UniqHash); ++i)
 116       if (UniqHash[i] != 0)
 117          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 118
 119    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 120         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 121       (*i)->ReMap(oldMap, newMap);
 122    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 123         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 124       (*i)->ReMap(oldMap, newMap);
 125    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 126         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 127       (*i)->ReMap(oldMap, newMap);
 128    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 129         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 130       (*i)->ReMap(oldMap, newMap);
 131    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 132         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 133       (*i)->ReMap(oldMap, newMap);
 134    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 135         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 136       (*i)->ReMap(oldMap, newMap);
 137    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 138         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 139       (*i)->ReMap(oldMap, newMap);
 140 }                                                                       /*}}}*/
 141 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 142 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 143                                         const unsigned long &Len) {
 144    void const * const oldMap = Map.Data();
 145    map_ptrloc const index = Map.WriteString(String, Len);
 146    if (index != 0)
 147       ReMap(oldMap, Map.Data());
 148    return index;
 149 }
 150                                                                         /*}}}*/
 151 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 152 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 153    void const * const oldMap = Map.Data();
 154    map_ptrloc const index = Map.WriteString(String);
 155    if (index != 0)
 156       ReMap(oldMap, Map.Data());
 157    return index;
 158 }
 159                                                                         /*}}}*/
 160 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 161    void const * const oldMap = Map.Data();
 162    map_ptrloc const index = Map.Allocate(size);
 163    if (index != 0)
 164       ReMap(oldMap, Map.Data());
 165    return index;
 166 }
 167                                                                         /*}}}*/
 168 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 169 // ---------------------------------------------------------------------
 170 /* This provides the generation of the entries in the cache. Each loop
 171    goes through a single package record from the underlying parse engine. */
 172 bool pkgCacheGenerator::MergeList(ListParser &List,
 173                                   pkgCache::VerIterator *OutVer)
 174 {
 175    List.Owner = this;
 176
 177    unsigned int Counter = 0;
 178    while (List.Step() == true)
 179    {
 180       string const PackageName = List.Package();
 181       if (PackageName.empty() == true)
 182          return false;
 183
 184       string const Arch = List.Architecture();
 185
 186       // Get a pointer to the package structure
 187       pkgCache::PkgIterator Pkg;
 188       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 189       if (NewPackage(Pkg, PackageName, Arch) == false)
 190          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 191       Counter++;
 192       if (Counter % 100 == 0 && Progress != 0)
 193          Progress->Progress(List.Offset());
 194
 195       /* Get a pointer to the version structure. We know the list is sorted
 196          so we use that fact in the search. Insertion of new versions is
 197          done with correct sorting */
 198       string Version = List.Version();
 199       if (Version.empty() == true)
 200       {
 201          // we first process the package, then the descriptions
 202          // (this has the bonus that we get MMap error when we run out
 203          //  of MMap space)
 204          pkgCache::VerIterator Ver(Cache);
 205          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 206          if (List.UsePackage(Pkg, Ver) == false)
 207             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 208                                  PackageName.c_str());
 209
 210          // Find the right version to write the description
 211          MD5SumValue CurMd5 = List.Description_md5();
 212          Ver = Pkg.VersionList();
 213
 214          for (; Ver.end() == false; ++Ver)
 215          {
 216             pkgCache::DescIterator Desc = Ver.DescriptionList();
 217             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 218             map_ptrloc *LastDesc = &Ver->DescriptionList;
 219             bool duplicate=false;
 220
 221             // don't add a new description if we have one for the given
 222             // md5 && language
 223             for ( ; Desc.end() == false; Desc++)
 224                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 225                    Desc.LanguageCode() == List.DescriptionLanguage())
 226                   duplicate=true;
 227             if(duplicate)
 228                continue;
 229
 230             for (Desc = Ver.DescriptionList();
 231                  Desc.end() == false;
 232                  LastDesc = &Desc->NextDesc, Desc++)
 233             {
 234                if (MD5SumValue(Desc.md5()) == CurMd5)
 235                {
 236                   // Add new description
 237                   void const * const oldMap = Map.Data();
 238                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 239                   if (oldMap != Map.Data())
 240                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 241                   *LastDesc = descindex;
 242                   Desc->ParentPkg = Pkg.Index();
 243
 244                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 245                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 246                   break;
 247                }
 248             }
 249          }
 250
 251          continue;
 252       }
 253
 254       pkgCache::VerIterator Ver = Pkg.VersionList();
 255       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 256       map_ptrloc *LastVer = &Pkg->VersionList;
 257       void const * oldMap = Map.Data();
 258       int Res = 1;
 259       unsigned long const Hash = List.VersionHash();
 260       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 261       {
 262          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 263          // Version is higher as current version - insert here
 264          if (Res > 0)
 265             break;
 266          // Versionstrings are equal - is hash also equal?
 267          if (Res == 0 && Ver->Hash == Hash)
 268             break;
 269          // proceed with the next till we have either the right
 270          // or we found another version (which will be lower)
 271       }
 272
 273       /* We already have a version for this item, record that we saw it */
 274       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 275       {
 276          if (List.UsePackage(Pkg,Ver) == false)
 277             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 278                                  PackageName.c_str());
 279
 280          if (NewFileVer(Ver,List) == false)
 281             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 282                                  PackageName.c_str());
 283
 284          // Read only a single record and return
 285          if (OutVer != 0)
 286          {
 287             *OutVer = Ver;
 288             FoundFileDeps |= List.HasFileDeps();
 289             return true;
 290          }
 291
 292          continue;
 293       }
 294
 295       // Add a new version
 296       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 297       if (verindex == 0 && _error->PendingError())
 298          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 299                               PackageName.c_str(), 1);
 300
 301       if (oldMap != Map.Data())
 302          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 303       *LastVer = verindex;
 304       Ver->ParentPkg = Pkg.Index();
 305       Ver->Hash = Hash;
 306
 307       if (List.NewVersion(Ver) == false)
 308          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 309                               PackageName.c_str(), 2);
 310
 311       if (List.UsePackage(Pkg,Ver) == false)
 312          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 313                               PackageName.c_str());
 314
 315       if (NewFileVer(Ver,List) == false)
 316          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 317                               PackageName.c_str(), 3);
 318
 319       // Read only a single record and return
 320       if (OutVer != 0)
 321       {
 322          *OutVer = Ver;
 323          FoundFileDeps |= List.HasFileDeps();
 324          return true;
 325       }
 326
 327       /* Record the Description data. Description data always exist in
 328          Packages and Translation-* files. */
 329       pkgCache::DescIterator Desc = Ver.DescriptionList();
 330       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 331       map_ptrloc *LastDesc = &Ver->DescriptionList;
 332
 333       // Skip to the end of description set
 334       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 335
 336       // Add new description
 337       oldMap = Map.Data();
 338       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 339       if (oldMap != Map.Data())
 340          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 341       *LastDesc = descindex;
 342       Desc->ParentPkg = Pkg.Index();
 343
 344       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 345          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 346    }
 347
 348    FoundFileDeps |= List.HasFileDeps();
 349
 350    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 351       return _error->Error(_("Wow, you exceeded the number of package "
 352                              "names this APT is capable of."));
 353    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 354       return _error->Error(_("Wow, you exceeded the number of versions "
 355                              "this APT is capable of."));
 356    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 357       return _error->Error(_("Wow, you exceeded the number of descriptions "
 358                              "this APT is capable of."));
 359    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 360       return _error->Error(_("Wow, you exceeded the number of dependencies "
 361                              "this APT is capable of."));
 362    return true;
 363 }
 364                                                                         /*}}}*/
 365 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 366 // ---------------------------------------------------------------------
 367 /* If we found any file depends while parsing the main list we need to
 368    resolve them. Since it is undesired to load the entire list of files
 369    into the cache as virtual packages we do a two stage effort. MergeList
 370    identifies the file depends and this creates Provdies for them by
 371    re-parsing all the indexs. */
 372 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 373 {
 374    List.Owner = this;
 375
 376    unsigned int Counter = 0;
 377    while (List.Step() == true)
 378    {
 379       string PackageName = List.Package();
 380       if (PackageName.empty() == true)
 381          return false;
 382       string Version = List.Version();
 383       if (Version.empty() == true)
 384          continue;
 385
 386       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 387       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 388       if (Pkg.end() == true)
 389          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 390                                 PackageName.c_str());
 391       Counter++;
 392       if (Counter % 100 == 0 && Progress != 0)
 393          Progress->Progress(List.Offset());
 394
 395       unsigned long Hash = List.VersionHash();
 396       pkgCache::VerIterator Ver = Pkg.VersionList();
 397       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 398       for (; Ver.end() == false; Ver++)
 399       {
 400          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 401          {
 402             if (List.CollectFileProvides(Cache,Ver) == false)
 403                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 404             break;
 405          }
 406       }
 407
 408       if (Ver.end() == true)
 409          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 410    }
 411
 412    return true;
 413 }
 414                                                                         /*}}}*/
 415 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 416 // ---------------------------------------------------------------------
 417 /* This creates a new group structure and adds it to the hash table */
 418 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 419 {
 420    Grp = Cache.FindGrp(Name);
 421    if (Grp.end() == false)
 422       return true;
 423
 424    // Get a structure
 425    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 426    if (unlikely(Group == 0))
 427       return false;
 428
 429    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 430    map_ptrloc const idxName = WriteStringInMap(Name);
 431    if (unlikely(idxName == 0))
 432       return false;
 433    Grp->Name = idxName;
 434
 435    // Insert it into the hash table
 436    unsigned long const Hash = Cache.Hash(Name);
 437    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 438    Cache.HeaderP->GrpHashTable[Hash] = Group;
 439
 440    Grp->ID = Cache.HeaderP->GroupCount++;
 441    return true;
 442 }
 443                                                                         /*}}}*/
 444 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 445 // ---------------------------------------------------------------------
 446 /* This creates a new package structure and adds it to the hash table */
 447 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 448                                         const string &Arch) {
 449    pkgCache::GrpIterator Grp;
 450    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 451    if (unlikely(NewGroup(Grp, Name) == false))
 452       return false;
 453
 454    Pkg = Grp.FindPkg(Arch);
 455       if (Pkg.end() == false)
 456          return true;
 457
 458    // Get a structure
 459    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 460    if (unlikely(Package == 0))
 461       return false;
 462    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 463
 464    // Insert the package into our package list
 465    if (Grp->FirstPackage == 0) // the group is new
 466    {
 467       // Insert it into the hash table
 468       unsigned long const Hash = Cache.Hash(Name);
 469       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 470       Cache.HeaderP->PkgHashTable[Hash] = Package;
 471       Grp->FirstPackage = Package;
 472    }
 473    else // Group the Packages together
 474    {
 475       // this package is the new last package
 476       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 477       Pkg->NextPackage = LastPkg->NextPackage;
 478       LastPkg->NextPackage = Package;
 479    }
 480    Grp->LastPackage = Package;
 481
 482    // Set the name, arch and the ID
 483    Pkg->Name = Grp->Name;
 484    Pkg->Group = Grp.Index();
 485    // all is mapped to the native architecture
 486    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 487    if (unlikely(idxArch == 0))
 488       return false;
 489    Pkg->Arch = idxArch;
 490    Pkg->ID = Cache.HeaderP->PackageCount++;
 491
 492    return true;
 493 }
 494                                                                         /*}}}*/
 495 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 496 // ---------------------------------------------------------------------
 497 /* */
 498 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 499                                    ListParser &List)
 500 {
 501    if (CurrentFile == 0)
 502       return true;
 503
 504    // Get a structure
 505    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 506    if (VerFile == 0)
 507       return 0;
 508
 509    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 510    VF->File = CurrentFile - Cache.PkgFileP;
 511
 512    // Link it to the end of the list
 513    map_ptrloc *Last = &Ver->FileList;
 514    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 515       Last = &V->NextFile;
 516    VF->NextFile = *Last;
 517    *Last = VF.Index();
 518
 519    VF->Offset = List.Offset();
 520    VF->Size = List.Size();
 521    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 522       Cache.HeaderP->MaxVerFileSize = VF->Size;
 523    Cache.HeaderP->VerFileCount++;
 524
 525    return true;
 526 }
 527                                                                         /*}}}*/
 528 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 529 // ---------------------------------------------------------------------
 530 /* This puts a version structure in the linked list */
 531 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 532                                             const string &VerStr,
 533                                             unsigned long Next)
 534 {
 535    // Get a structure
 536    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 537    if (Version == 0)
 538       return 0;
 539
 540    // Fill it in
 541    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 542    Ver->NextVer = Next;
 543    Ver->ID = Cache.HeaderP->VersionCount++;
 544    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 545    if (unlikely(idxVerStr == 0))
 546       return 0;
 547    Ver->VerStr = idxVerStr;
 548
 549    return Version;
 550 }
 551                                                                         /*}}}*/
 552 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* */
 555 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 556                                    ListParser &List)
 557 {
 558    if (CurrentFile == 0)
 559       return true;
 560
 561    // Get a structure
 562    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 563    if (DescFile == 0)
 564       return false;
 565
 566    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 567    DF->File = CurrentFile - Cache.PkgFileP;
 568
 569    // Link it to the end of the list
 570    map_ptrloc *Last = &Desc->FileList;
 571    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 572       Last = &D->NextFile;
 573
 574    DF->NextFile = *Last;
 575    *Last = DF.Index();
 576
 577    DF->Offset = List.Offset();
 578    DF->Size = List.Size();
 579    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 580       Cache.HeaderP->MaxDescFileSize = DF->Size;
 581    Cache.HeaderP->DescFileCount++;
 582
 583    return true;
 584 }
 585                                                                         /*}}}*/
 586 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 587 // ---------------------------------------------------------------------
 588 /* This puts a description structure in the linked list */
 589 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 590                                             const string &Lang,
 591                                             const MD5SumValue &md5sum,
 592                                             map_ptrloc Next)
 593 {
 594    // Get a structure
 595    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 596    if (Description == 0)
 597       return 0;
 598
 599    // Fill it in
 600    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 601    Desc->NextDesc = Next;
 602    Desc->ID = Cache.HeaderP->DescriptionCount++;
 603    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 604    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 605    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 606       return 0;
 607    Desc->language_code = idxlanguage_code;
 608    Desc->md5sum = idxmd5sum;
 609
 610    return Description;
 611 }
 612                                                                         /*}}}*/
 613 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 614 // ---------------------------------------------------------------------
 615 /* This prepares the Cache for delivery */
 616 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 617 {
 618    // FIXME: add progress reporting for this operation
 619    // Do we have different architectures in your groups ?
 620    vector<string> archs = APT::Configuration::getArchitectures();
 621    if (archs.size() > 1)
 622    {
 623       // Create Conflicts in between the group
 624       pkgCache::GrpIterator G = GetCache().GrpBegin();
 625       Dynamic<pkgCache::GrpIterator> DynG(G);
 626       for (; G.end() != true; G++)
 627       {
 628          string const PkgName = G.Name();
 629          pkgCache::PkgIterator P = G.PackageList();
 630          Dynamic<pkgCache::PkgIterator> DynP(P);
 631          for (; P.end() != true; P = G.NextPkg(P))
 632          {
 633             pkgCache::PkgIterator allPkg;
 634             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 635             pkgCache::VerIterator V = P.VersionList();
 636             Dynamic<pkgCache::VerIterator> DynV(V);
 637             for (; V.end() != true; V++)
 638             {
 639                // copy P.Arch() into a string here as a cache remap
 640                // in NewDepends() later may alter the pointer location
 641                string Arch = P.Arch() == NULL ? "" : P.Arch();
 642                map_ptrloc *OldDepLast = NULL;
 643                /* MultiArch handling introduces a lot of implicit Dependencies:
 644                 - MultiArch: same → Co-Installable if they have the same version
 645                 - Architecture: all → Need to be Co-Installable for internal reasons
 646                 - All others conflict with all other group members */
 647                bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 648                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 649                {
 650                   if (*A == Arch)
 651                      continue;
 652                   /* We allow only one installed arch at the time
 653                      per group, therefore each group member conflicts
 654                      with all other group members */
 655                   pkgCache::PkgIterator D = G.FindPkg(*A);
 656                   Dynamic<pkgCache::PkgIterator> DynD(D);
 657                   if (D.end() == true)
 658                      continue;
 659                   if (coInstall == true)
 660                   {
 661                      // Replaces: ${self}:other ( << ${binary:Version})
 662                      NewDepends(D, V, V.VerStr(),
 663                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 664                                 OldDepLast);
 665                      // Breaks: ${self}:other (!= ${binary:Version})
 666                      NewDepends(D, V, V.VerStr(),
 667                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 668                                 OldDepLast);
 669                   } else {
 670                         // Conflicts: ${self}:other
 671                         NewDepends(D, V, "",
 672                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 673                                    OldDepLast);
 674                   }
 675                }
 676             }
 677          }
 678       }
 679    }
 680    return true;
 681 }
 682                                                                         /*}}}*/
 683 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 684 // ---------------------------------------------------------------------
 685 /* This creates a dependency element in the tree. It is linked to the
 686    version and to the package that it is pointing to. */
 687 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 688                                    pkgCache::VerIterator &Ver,
 689                                    string const &Version,
 690                                    unsigned int const &Op,
 691                                    unsigned int const &Type,
 692                                    map_ptrloc* &OldDepLast)
 693 {
 694    void const * const oldMap = Map.Data();
 695    // Get a structure
 696    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 697    if (unlikely(Dependency == 0))
 698       return false;
 699
 700    // Fill it in
 701    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 702    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 703    Dep->ParentVer = Ver.Index();
 704    Dep->Type = Type;
 705    Dep->CompareOp = Op;
 706    Dep->ID = Cache.HeaderP->DependsCount++;
 707
 708    // Probe the reverse dependency list for a version string that matches
 709    if (Version.empty() == false)
 710    {
 711 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 712          if (I->Version != 0 && I.TargetVer() == Version)
 713             Dep->Version = I->Version;*/
 714       if (Dep->Version == 0) {
 715          map_ptrloc const index = WriteStringInMap(Version);
 716          if (unlikely(index == 0))
 717             return false;
 718          Dep->Version = index;
 719       }
 720    }
 721
 722    // Link it to the package
 723    Dep->Package = Pkg.Index();
 724    Dep->NextRevDepends = Pkg->RevDepends;
 725    Pkg->RevDepends = Dep.Index();
 726
 727    // Do we know where to link the Dependency to?
 728    if (OldDepLast == NULL)
 729    {
 730       OldDepLast = &Ver->DependsList;
 731       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 732          OldDepLast = &D->NextDepends;
 733    } else if (oldMap != Map.Data())
 734       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 735
 736    Dep->NextDepends = *OldDepLast;
 737    *OldDepLast = Dep.Index();
 738    OldDepLast = &Dep->NextDepends;
 739
 740    return true;
 741 }
 742                                                                         /*}}}*/
 743 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 744 // ---------------------------------------------------------------------
 745 /* This creates a Group and the Package to link this dependency to if
 746    needed and handles also the caching of the old endpoint */
 747 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 748                                                const string &PackageName,
 749                                                const string &Arch,
 750                                                const string &Version,
 751                                                unsigned int Op,
 752                                                unsigned int Type)
 753 {
 754    pkgCache::GrpIterator Grp;
 755    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 756    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 757       return false;
 758
 759    // Locate the target package
 760    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 761    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 762    if (Pkg.end() == true) {
 763       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 764          return false;
 765    }
 766
 767    // Is it a file dependency?
 768    if (unlikely(PackageName[0] == '/'))
 769       FoundFileDeps = true;
 770
 771    /* Caching the old end point speeds up generation substantially */
 772    if (OldDepVer != Ver) {
 773       OldDepLast = NULL;
 774       OldDepVer = Ver;
 775    }
 776
 777    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 778 }
 779                                                                         /*}}}*/
 780 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 781 // ---------------------------------------------------------------------
 782 /* */
 783 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 784                                                 const string &PkgName,
 785                                                 const string &PkgArch,
 786                                                 const string &Version)
 787 {
 788    pkgCache &Cache = Owner->Cache;
 789
 790    // We do not add self referencing provides
 791    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 792         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 793       return true;
 794
 795    // Get a structure
 796    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 797    if (unlikely(Provides == 0))
 798       return false;
 799    Cache.HeaderP->ProvidesCount++;
 800
 801    // Fill it in
 802    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 803    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 804    Prv->Version = Ver.Index();
 805    Prv->NextPkgProv = Ver->ProvidesList;
 806    Ver->ProvidesList = Prv.Index();
 807    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 808       return false;
 809
 810    // Locate the target package
 811    pkgCache::PkgIterator Pkg;
 812    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 813    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 814       return false;
 815
 816    // Link it to the package
 817    Prv->ParentPkg = Pkg.Index();
 818    Prv->NextProvides = Pkg->ProvidesList;
 819    Pkg->ProvidesList = Prv.Index();
 820
 821    return true;
 822 }
 823                                                                         /*}}}*/
 824 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 825 // ---------------------------------------------------------------------
 826 /* This is used to select which file is to be associated with all newly
 827    added versions. The caller is responsible for setting the IMS fields. */
 828 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 829                                    const pkgIndexFile &Index,
 830                                    unsigned long Flags)
 831 {
 832    // Get some space for the structure
 833    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 834    if (unlikely(idxFile == 0))
 835       return false;
 836    CurrentFile = Cache.PkgFileP + idxFile;
 837
 838    // Fill it in
 839    map_ptrloc const idxFileName = WriteStringInMap(File);
 840    map_ptrloc const idxSite = WriteUniqString(Site);
 841    if (unlikely(idxFileName == 0 || idxSite == 0))
 842       return false;
 843    CurrentFile->FileName = idxFileName;
 844    CurrentFile->Site = idxSite;
 845    CurrentFile->NextFile = Cache.HeaderP->FileList;
 846    CurrentFile->Flags = Flags;
 847    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 848    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 849    if (unlikely(idxIndexType == 0))
 850       return false;
 851    CurrentFile->IndexType = idxIndexType;
 852    PkgFileName = File;
 853    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 854    Cache.HeaderP->PackageFileCount++;
 855
 856    if (Progress != 0)
 857       Progress->SubProgress(Index.Size());
 858    return true;
 859 }
 860                                                                         /*}}}*/
 861 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 862 // ---------------------------------------------------------------------
 863 /* This is used to create handles to strings. Given the same text it
 864    always returns the same number */
 865 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 866                                                  unsigned int Size)
 867 {
 868    /* We use a very small transient hash table here, this speeds up generation
 869       by a fair amount on slower machines */
 870    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 871    if (Bucket != 0 &&
 872        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 873       return Bucket->String;
 874
 875    // Search for an insertion point
 876    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 877    int Res = 1;
 878    map_ptrloc *Last = &Cache.HeaderP->StringList;
 879    for (; I != Cache.StringItemP; Last = &I->NextItem,
 880         I = Cache.StringItemP + I->NextItem)
 881    {
 882       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 883       if (Res >= 0)
 884          break;
 885    }
 886
 887    // Match
 888    if (Res == 0)
 889    {
 890       Bucket = I;
 891       return I->String;
 892    }
 893
 894    // Get a structure
 895    void const * const oldMap = Map.Data();
 896    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 897    if (Item == 0)
 898       return 0;
 899
 900    map_ptrloc const idxString = WriteStringInMap(S,Size);
 901    if (unlikely(idxString == 0))
 902       return 0;
 903    if (oldMap != Map.Data()) {
 904       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 905       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 906    }
 907    *Last = Item;
 908
 909    // Fill in the structure
 910    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 911    ItemP->NextItem = I - Cache.StringItemP;
 912    ItemP->String = idxString;
 913
 914    Bucket = ItemP;
 915    return ItemP->String;
 916 }
 917                                                                         /*}}}*/
 918 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 919 // ---------------------------------------------------------------------
 920 /* This just verifies that each file in the list of index files exists,
 921    has matching attributes with the cache and the cache does not have
 922    any extra files. */
 923 static bool CheckValidity(const string &CacheFile,
 924                           pkgSourceList &List,
 925                           FileIterator Start,
 926                           FileIterator End,
 927                           MMap **OutMap = 0)
 928 {
 929    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 930    // No file, certainly invalid
 931    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 932    {
 933       if (Debug == true)
 934          std::clog << "CacheFile doesn't exist" << std::endl;
 935       return false;
 936    }
 937
 938    if (List.GetLastModifiedTime() < GetModificationTime(CacheFile))
 939    {
 940       if (Debug == true)
 941          std::clog << "sources.list is newer than the cache" << std::endl;
 942       return false;
 943    }
 944
 945    // Map it
 946    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 947    SPtr<MMap> Map = new MMap(CacheF,0);
 948    pkgCache Cache(Map);
 949    if (_error->PendingError() == true || Map->Size() == 0)
 950    {
 951       if (Debug == true)
 952          std::clog << "Errors are pending or Map is empty()" << std::endl;
 953       _error->Discard();
 954       return false;
 955    }
 956
 957    /* Now we check every index file, see if it is in the cache,
 958       verify the IMS data and check that it is on the disk too.. */
 959    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 960    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 961    for (; Start != End; Start++)
 962    {
 963       if (Debug == true)
 964          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 965       if ((*Start)->HasPackages() == false)
 966       {
 967          if (Debug == true)
 968             std::clog << "Has NO packages" << std::endl;
 969          continue;
 970       }
 971
 972       if ((*Start)->Exists() == false)
 973       {
 974 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 975          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 976                           (*Start)->Describe().c_str());
 977 #endif
 978          if (Debug == true)
 979             std::clog << "file doesn't exist" << std::endl;
 980          continue;
 981       }
 982
 983       // FindInCache is also expected to do an IMS check.
 984       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 985       if (File.end() == true)
 986       {
 987          if (Debug == true)
 988             std::clog << "FindInCache returned end-Pointer" << std::endl;
 989          return false;
 990       }
 991
 992       Visited[File->ID] = true;
 993       if (Debug == true)
 994          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 995    }
 996
 997    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 998       if (Visited[I] == false)
 999       {
1000          if (Debug == true)
1001             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1002          return false;
1003       }
1004
1005    if (_error->PendingError() == true)
1006    {
1007       if (Debug == true)
1008       {
1009          std::clog << "Validity failed because of pending errors:" << std::endl;
1010          _error->DumpErrors();
1011       }
1012       _error->Discard();
1013       return false;
1014    }
1015
1016    if (OutMap != 0)
1017       *OutMap = Map.UnGuard();
1018    return true;
1019 }
1020                                                                         /*}}}*/
1021 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1022 // ---------------------------------------------------------------------
1023 /* Size is kind of an abstract notion that is only used for the progress
1024    meter */
1025 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1026 {
1027    unsigned long TotalSize = 0;
1028    for (; Start != End; Start++)
1029    {
1030       if ((*Start)->HasPackages() == false)
1031          continue;
1032       TotalSize += (*Start)->Size();
1033    }
1034    return TotalSize;
1035 }
1036                                                                         /*}}}*/
1037 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1038 // ---------------------------------------------------------------------
1039 /* */
1040 static bool BuildCache(pkgCacheGenerator &Gen,
1041                        OpProgress *Progress,
1042                        unsigned long &CurrentSize,unsigned long TotalSize,
1043                        FileIterator Start, FileIterator End)
1044 {
1045    FileIterator I;
1046    for (I = Start; I != End; I++)
1047    {
1048       if ((*I)->HasPackages() == false)
1049          continue;
1050
1051       if ((*I)->Exists() == false)
1052          continue;
1053
1054       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1055       {
1056          _error->Warning("Duplicate sources.list entry %s",
1057                          (*I)->Describe().c_str());
1058          continue;
1059       }
1060
1061       unsigned long Size = (*I)->Size();
1062       if (Progress != NULL)
1063          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1064       CurrentSize += Size;
1065
1066       if ((*I)->Merge(Gen,Progress) == false)
1067          return false;
1068    }
1069
1070    if (Gen.HasFileDeps() == true)
1071    {
1072       if (Progress != NULL)
1073          Progress->Done();
1074       TotalSize = ComputeSize(Start, End);
1075       CurrentSize = 0;
1076       for (I = Start; I != End; I++)
1077       {
1078          unsigned long Size = (*I)->Size();
1079          if (Progress != NULL)
1080             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1081          CurrentSize += Size;
1082          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1083             return false;
1084       }
1085    }
1086
1087    return true;
1088 }
1089                                                                         /*}}}*/
1090 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1091 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1092    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1093    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1094    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1095    Flags |= MMap::Moveable;
1096    if (_config->FindB("APT::Cache-Fallback", false) == true)
1097       Flags |= MMap::Fallback;
1098    if (CacheF != NULL)
1099       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1100    else
1101       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1102 }
1103                                                                         /*}}}*/
1104 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1105 // ---------------------------------------------------------------------
1106 /* This makes sure that the status cache (the cache that has all
1107    index files from the sources list and all local ones) is ready
1108    to be mmaped. If OutMap is not zero then a MMap object representing
1109    the cache will be stored there. This is pretty much mandetory if you
1110    are using AllowMem. AllowMem lets the function be run as non-root
1111    where it builds the cache 'fast' into a memory buffer. */
1112 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1113                         MMap **OutMap, bool AllowMem)
1114    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1115 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1116                         MMap **OutMap,bool AllowMem)
1117 {
1118    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1119
1120    vector<pkgIndexFile *> Files;
1121    for (vector<metaIndex *>::const_iterator i = List.begin();
1122         i != List.end();
1123         i++)
1124    {
1125       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1126       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1127            j != Indexes->end();
1128            j++)
1129          Files.push_back (*j);
1130    }
1131
1132    unsigned long const EndOfSource = Files.size();
1133    if (_system->AddStatusFiles(Files) == false)
1134       return false;
1135
1136    // Decide if we can write to the files..
1137    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1138    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1139
1140    // ensure the cache directory exists
1141    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1142    {
1143       string dir = _config->FindDir("Dir::Cache");
1144       size_t const len = dir.size();
1145       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1146          dir = dir.substr(0, len - 5);
1147       if (CacheFile.empty() == false)
1148          CreateDirectory(dir, flNotFile(CacheFile));
1149       if (SrcCacheFile.empty() == false)
1150          CreateDirectory(dir, flNotFile(SrcCacheFile));
1151    }
1152
1153    // Decide if we can write to the cache
1154    bool Writeable = false;
1155    if (CacheFile.empty() == false)
1156       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1157    else
1158       if (SrcCacheFile.empty() == false)
1159          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1160    if (Debug == true)
1161       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1162
1163    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1164       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1165
1166    if (Progress != NULL)
1167       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1168
1169    // Cache is OK, Fin.
1170    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1171    {
1172       if (Progress != NULL)
1173          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1174       if (Debug == true)
1175          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1176       return true;
1177    }
1178    else if (Debug == true)
1179          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1180
1181    /* At this point we know we need to reconstruct the package cache,
1182       begin. */
1183    SPtr<FileFd> CacheF;
1184    SPtr<DynamicMMap> Map;
1185    if (Writeable == true && CacheFile.empty() == false)
1186    {
1187       _error->PushToStack();
1188       unlink(CacheFile.c_str());
1189       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1190       fchmod(CacheF->Fd(),0644);
1191       Map = CreateDynamicMMap(CacheF, MMap::Public);
1192       if (_error->PendingError() == true)
1193       {
1194          delete CacheF.UnGuard();
1195          delete Map.UnGuard();
1196          if (Debug == true)
1197             std::clog << "Open filebased MMap FAILED" << std::endl;
1198          Writeable = false;
1199          if (AllowMem == false)
1200          {
1201             _error->MergeWithStack();
1202             return false;
1203          }
1204          _error->RevertToStack();
1205       }
1206       else if (Debug == true)
1207       {
1208          _error->MergeWithStack();
1209          std::clog << "Open filebased MMap" << std::endl;
1210       }
1211    }
1212    if (Writeable == false || CacheFile.empty() == true)
1213    {
1214       // Just build it in memory..
1215       Map = CreateDynamicMMap(NULL);
1216       if (Debug == true)
1217          std::clog << "Open memory Map (not filebased)" << std::endl;
1218    }
1219
1220    // Lets try the source cache.
1221    unsigned long CurrentSize = 0;
1222    unsigned long TotalSize = 0;
1223    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1224                      Files.begin()+EndOfSource) == true)
1225    {
1226       if (Debug == true)
1227          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1228       // Preload the map with the source cache
1229       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1230       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1231       if ((alloc == 0 && _error->PendingError())
1232                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1233                                 SCacheF.Size()) == false)
1234          return false;
1235
1236       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1237
1238       // Build the status cache
1239       pkgCacheGenerator Gen(Map.Get(),Progress);
1240       if (_error->PendingError() == true)
1241          return false;
1242       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1243                      Files.begin()+EndOfSource,Files.end()) == false)
1244          return false;
1245
1246       // FIXME: move me to a better place
1247       Gen.FinishCache(Progress);
1248    }
1249    else
1250    {
1251       if (Debug == true)
1252          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1253       TotalSize = ComputeSize(Files.begin(),Files.end());
1254
1255       // Build the source cache
1256       pkgCacheGenerator Gen(Map.Get(),Progress);
1257       if (_error->PendingError() == true)
1258          return false;
1259       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1260                      Files.begin(),Files.begin()+EndOfSource) == false)
1261          return false;
1262
1263       // Write it back
1264       if (Writeable == true && SrcCacheFile.empty() == false)
1265       {
1266          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1267          if (_error->PendingError() == true)
1268             return false;
1269
1270          fchmod(SCacheF.Fd(),0644);
1271
1272          // Write out the main data
1273          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1274             return _error->Error(_("IO Error saving source cache"));
1275          SCacheF.Sync();
1276
1277          // Write out the proper header
1278          Gen.GetCache().HeaderP->Dirty = false;
1279          if (SCacheF.Seek(0) == false ||
1280              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1281             return _error->Error(_("IO Error saving source cache"));
1282          Gen.GetCache().HeaderP->Dirty = true;
1283          SCacheF.Sync();
1284       }
1285
1286       // Build the status cache
1287       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1288                      Files.begin()+EndOfSource,Files.end()) == false)
1289          return false;
1290
1291       // FIXME: move me to a better place
1292       Gen.FinishCache(Progress);
1293    }
1294    if (Debug == true)
1295       std::clog << "Caches are ready for shipping" << std::endl;
1296
1297    if (_error->PendingError() == true)
1298       return false;
1299    if (OutMap != 0)
1300    {
1301       if (CacheF != 0)
1302       {
1303          delete Map.UnGuard();
1304          *OutMap = new MMap(*CacheF,0);
1305       }
1306       else
1307       {
1308          *OutMap = Map.UnGuard();
1309       }
1310    }
1311
1312    return true;
1313 }
1314                                                                         /*}}}*/
1315 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1316 // ---------------------------------------------------------------------
1317 /* */
1318 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1319    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1320 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1321 {
1322    vector<pkgIndexFile *> Files;
1323    unsigned long EndOfSource = Files.size();
1324    if (_system->AddStatusFiles(Files) == false)
1325       return false;
1326
1327    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1328    unsigned long CurrentSize = 0;
1329    unsigned long TotalSize = 0;
1330
1331    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1332
1333    // Build the status cache
1334    if (Progress != NULL)
1335       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1336    pkgCacheGenerator Gen(Map.Get(),Progress);
1337    if (_error->PendingError() == true)
1338       return false;
1339    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1340                   Files.begin()+EndOfSource,Files.end()) == false)
1341       return false;
1342
1343    // FIXME: move me to a better place
1344    Gen.FinishCache(Progress);
1345
1346    if (_error->PendingError() == true)
1347       return false;
1348    *OutMap = Map.UnGuard();
1349
1350    return true;
1351 }
1352                                                                         /*}}}*/