apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26
  27 #include <apt-pkg/tagfile.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Cache.HeaderP->CacheFileSize = Map.Size();
 102    Map.Sync(0,sizeof(pkgCache::Header));
 103 }
 104                                                                         /*}}}*/
 105 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 106    if (oldMap == newMap)
 107       return;
 108
 109    if (_config->FindB("Debug::pkgCacheGen", false))
 110       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 111
 112    Cache.ReMap(false);
 113
 114    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 115
 116    for (size_t i = 0; i < _count(UniqHash); ++i)
 117       if (UniqHash[i] != 0)
 118          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 119
 120    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 121         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 122       (*i)->ReMap(oldMap, newMap);
 123    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 124         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 125       (*i)->ReMap(oldMap, newMap);
 126    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 127         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 128       (*i)->ReMap(oldMap, newMap);
 129    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 130         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 131       (*i)->ReMap(oldMap, newMap);
 132    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 133         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 134       (*i)->ReMap(oldMap, newMap);
 135    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 136         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 137       (*i)->ReMap(oldMap, newMap);
 138    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 139         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 140       (*i)->ReMap(oldMap, newMap);
 141 }                                                                       /*}}}*/
 142 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 143 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 144                                         const unsigned long &Len) {
 145    void const * const oldMap = Map.Data();
 146    map_ptrloc const index = Map.WriteString(String, Len);
 147    if (index != 0)
 148       ReMap(oldMap, Map.Data());
 149    return index;
 150 }
 151                                                                         /*}}}*/
 152 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 153 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 154    void const * const oldMap = Map.Data();
 155    map_ptrloc const index = Map.WriteString(String);
 156    if (index != 0)
 157       ReMap(oldMap, Map.Data());
 158    return index;
 159 }
 160                                                                         /*}}}*/
 161 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 162    void const * const oldMap = Map.Data();
 163    map_ptrloc const index = Map.Allocate(size);
 164    if (index != 0)
 165       ReMap(oldMap, Map.Data());
 166    return index;
 167 }
 168                                                                         /*}}}*/
 169 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 170 // ---------------------------------------------------------------------
 171 /* This provides the generation of the entries in the cache. Each loop
 172    goes through a single package record from the underlying parse engine. */
 173 bool pkgCacheGenerator::MergeList(ListParser &List,
 174                                   pkgCache::VerIterator *OutVer)
 175 {
 176    List.Owner = this;
 177
 178    unsigned int Counter = 0;
 179    while (List.Step() == true)
 180    {
 181       string const PackageName = List.Package();
 182       if (PackageName.empty() == true)
 183          return false;
 184
 185       string const Arch = List.Architecture();
 186
 187       // Get a pointer to the package structure
 188       pkgCache::PkgIterator Pkg;
 189       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 190       if (NewPackage(Pkg, PackageName, Arch) == false)
 191          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 192       Counter++;
 193       if (Counter % 100 == 0 && Progress != 0)
 194          Progress->Progress(List.Offset());
 195
 196       /* Get a pointer to the version structure. We know the list is sorted
 197          so we use that fact in the search. Insertion of new versions is
 198          done with correct sorting */
 199       string Version = List.Version();
 200       if (Version.empty() == true)
 201       {
 202          // we first process the package, then the descriptions
 203          // (this has the bonus that we get MMap error when we run out
 204          //  of MMap space)
 205          pkgCache::VerIterator Ver(Cache);
 206          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 207          if (List.UsePackage(Pkg, Ver) == false)
 208             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 209                                  PackageName.c_str());
 210
 211          // Find the right version to write the description
 212          MD5SumValue CurMd5 = List.Description_md5();
 213          Ver = Pkg.VersionList();
 214
 215          for (; Ver.end() == false; ++Ver)
 216          {
 217             pkgCache::DescIterator Desc = Ver.DescriptionList();
 218             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 219             map_ptrloc *LastDesc = &Ver->DescriptionList;
 220             bool duplicate=false;
 221
 222             // don't add a new description if we have one for the given
 223             // md5 && language
 224             for ( ; Desc.end() == false; Desc++)
 225                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 226                    Desc.LanguageCode() == List.DescriptionLanguage())
 227                   duplicate=true;
 228             if(duplicate)
 229                continue;
 230
 231             for (Desc = Ver.DescriptionList();
 232                  Desc.end() == false;
 233                  LastDesc = &Desc->NextDesc, Desc++)
 234             {
 235                if (MD5SumValue(Desc.md5()) == CurMd5)
 236                {
 237                   // Add new description
 238                   void const * const oldMap = Map.Data();
 239                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 240                   if (oldMap != Map.Data())
 241                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 242                   *LastDesc = descindex;
 243                   Desc->ParentPkg = Pkg.Index();
 244
 245                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 246                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 247                   break;
 248                }
 249             }
 250          }
 251
 252          continue;
 253       }
 254
 255       pkgCache::VerIterator Ver = Pkg.VersionList();
 256       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 257       map_ptrloc *LastVer = &Pkg->VersionList;
 258       void const * oldMap = Map.Data();
 259       int Res = 1;
 260       unsigned long const Hash = List.VersionHash();
 261       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 262       {
 263          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 264          // Version is higher as current version - insert here
 265          if (Res > 0)
 266             break;
 267          // Versionstrings are equal - is hash also equal?
 268          if (Res == 0 && Ver->Hash == Hash)
 269             break;
 270          // proceed with the next till we have either the right
 271          // or we found another version (which will be lower)
 272       }
 273
 274       /* We already have a version for this item, record that we saw it */
 275       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 276       {
 277          if (List.UsePackage(Pkg,Ver) == false)
 278             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 279                                  PackageName.c_str());
 280
 281          if (NewFileVer(Ver,List) == false)
 282             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 283                                  PackageName.c_str());
 284
 285          // Read only a single record and return
 286          if (OutVer != 0)
 287          {
 288             *OutVer = Ver;
 289             FoundFileDeps |= List.HasFileDeps();
 290             return true;
 291          }
 292
 293          continue;
 294       }
 295
 296       // Add a new version
 297       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 298       if (verindex == 0 && _error->PendingError())
 299          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 300                               PackageName.c_str(), 1);
 301
 302       if (oldMap != Map.Data())
 303          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 304       *LastVer = verindex;
 305       Ver->ParentPkg = Pkg.Index();
 306       Ver->Hash = Hash;
 307
 308       if (List.NewVersion(Ver) == false)
 309          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 310                               PackageName.c_str(), 2);
 311
 312       if (List.UsePackage(Pkg,Ver) == false)
 313          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 314                               PackageName.c_str());
 315
 316       if (NewFileVer(Ver,List) == false)
 317          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 318                               PackageName.c_str(), 3);
 319
 320       // Read only a single record and return
 321       if (OutVer != 0)
 322       {
 323          *OutVer = Ver;
 324          FoundFileDeps |= List.HasFileDeps();
 325          return true;
 326       }
 327
 328       /* Record the Description data. Description data always exist in
 329          Packages and Translation-* files. */
 330       pkgCache::DescIterator Desc = Ver.DescriptionList();
 331       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 332       map_ptrloc *LastDesc = &Ver->DescriptionList;
 333
 334       // Skip to the end of description set
 335       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 336
 337       // Add new description
 338       oldMap = Map.Data();
 339       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 340       if (oldMap != Map.Data())
 341          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 342       *LastDesc = descindex;
 343       Desc->ParentPkg = Pkg.Index();
 344
 345       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 346          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 347    }
 348
 349    FoundFileDeps |= List.HasFileDeps();
 350
 351    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 352       return _error->Error(_("Wow, you exceeded the number of package "
 353                              "names this APT is capable of."));
 354    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 355       return _error->Error(_("Wow, you exceeded the number of versions "
 356                              "this APT is capable of."));
 357    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 358       return _error->Error(_("Wow, you exceeded the number of descriptions "
 359                              "this APT is capable of."));
 360    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 361       return _error->Error(_("Wow, you exceeded the number of dependencies "
 362                              "this APT is capable of."));
 363    return true;
 364 }
 365                                                                         /*}}}*/
 366 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 367 // ---------------------------------------------------------------------
 368 /* If we found any file depends while parsing the main list we need to
 369    resolve them. Since it is undesired to load the entire list of files
 370    into the cache as virtual packages we do a two stage effort. MergeList
 371    identifies the file depends and this creates Provdies for them by
 372    re-parsing all the indexs. */
 373 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 374 {
 375    List.Owner = this;
 376
 377    unsigned int Counter = 0;
 378    while (List.Step() == true)
 379    {
 380       string PackageName = List.Package();
 381       if (PackageName.empty() == true)
 382          return false;
 383       string Version = List.Version();
 384       if (Version.empty() == true)
 385          continue;
 386
 387       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 388       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 389       if (Pkg.end() == true)
 390          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 391                                 PackageName.c_str());
 392       Counter++;
 393       if (Counter % 100 == 0 && Progress != 0)
 394          Progress->Progress(List.Offset());
 395
 396       unsigned long Hash = List.VersionHash();
 397       pkgCache::VerIterator Ver = Pkg.VersionList();
 398       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 399       for (; Ver.end() == false; Ver++)
 400       {
 401          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 402          {
 403             if (List.CollectFileProvides(Cache,Ver) == false)
 404                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 405             break;
 406          }
 407       }
 408
 409       if (Ver.end() == true)
 410          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 411    }
 412
 413    return true;
 414 }
 415                                                                         /*}}}*/
 416 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 417 // ---------------------------------------------------------------------
 418 /* This creates a new group structure and adds it to the hash table */
 419 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 420 {
 421    Grp = Cache.FindGrp(Name);
 422    if (Grp.end() == false)
 423       return true;
 424
 425    // Get a structure
 426    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 427    if (unlikely(Group == 0))
 428       return false;
 429
 430    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 431    map_ptrloc const idxName = WriteStringInMap(Name);
 432    if (unlikely(idxName == 0))
 433       return false;
 434    Grp->Name = idxName;
 435
 436    // Insert it into the hash table
 437    unsigned long const Hash = Cache.Hash(Name);
 438    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 439    Cache.HeaderP->GrpHashTable[Hash] = Group;
 440
 441    Grp->ID = Cache.HeaderP->GroupCount++;
 442    return true;
 443 }
 444                                                                         /*}}}*/
 445 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 446 // ---------------------------------------------------------------------
 447 /* This creates a new package structure and adds it to the hash table */
 448 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 449                                         const string &Arch) {
 450    pkgCache::GrpIterator Grp;
 451    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 452    if (unlikely(NewGroup(Grp, Name) == false))
 453       return false;
 454
 455    Pkg = Grp.FindPkg(Arch);
 456       if (Pkg.end() == false)
 457          return true;
 458
 459    // Get a structure
 460    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 461    if (unlikely(Package == 0))
 462       return false;
 463    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 464
 465    // Insert the package into our package list
 466    if (Grp->FirstPackage == 0) // the group is new
 467    {
 468       // Insert it into the hash table
 469       unsigned long const Hash = Cache.Hash(Name);
 470       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 471       Cache.HeaderP->PkgHashTable[Hash] = Package;
 472       Grp->FirstPackage = Package;
 473    }
 474    else // Group the Packages together
 475    {
 476       // this package is the new last package
 477       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 478       Pkg->NextPackage = LastPkg->NextPackage;
 479       LastPkg->NextPackage = Package;
 480    }
 481    Grp->LastPackage = Package;
 482
 483    // Set the name, arch and the ID
 484    Pkg->Name = Grp->Name;
 485    Pkg->Group = Grp.Index();
 486    // all is mapped to the native architecture
 487    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 488    if (unlikely(idxArch == 0))
 489       return false;
 490    Pkg->Arch = idxArch;
 491    Pkg->ID = Cache.HeaderP->PackageCount++;
 492
 493    return true;
 494 }
 495                                                                         /*}}}*/
 496 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* */
 499 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 500                                    ListParser &List)
 501 {
 502    if (CurrentFile == 0)
 503       return true;
 504
 505    // Get a structure
 506    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 507    if (VerFile == 0)
 508       return 0;
 509
 510    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 511    VF->File = CurrentFile - Cache.PkgFileP;
 512
 513    // Link it to the end of the list
 514    map_ptrloc *Last = &Ver->FileList;
 515    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 516       Last = &V->NextFile;
 517    VF->NextFile = *Last;
 518    *Last = VF.Index();
 519
 520    VF->Offset = List.Offset();
 521    VF->Size = List.Size();
 522    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 523       Cache.HeaderP->MaxVerFileSize = VF->Size;
 524    Cache.HeaderP->VerFileCount++;
 525
 526    return true;
 527 }
 528                                                                         /*}}}*/
 529 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* This puts a version structure in the linked list */
 532 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 533                                             const string &VerStr,
 534                                             unsigned long Next)
 535 {
 536    // Get a structure
 537    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 538    if (Version == 0)
 539       return 0;
 540
 541    // Fill it in
 542    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 543    Ver->NextVer = Next;
 544    Ver->ID = Cache.HeaderP->VersionCount++;
 545    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 546    if (unlikely(idxVerStr == 0))
 547       return 0;
 548    Ver->VerStr = idxVerStr;
 549
 550    return Version;
 551 }
 552                                                                         /*}}}*/
 553 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 554 // ---------------------------------------------------------------------
 555 /* */
 556 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 557                                    ListParser &List)
 558 {
 559    if (CurrentFile == 0)
 560       return true;
 561
 562    // Get a structure
 563    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 564    if (DescFile == 0)
 565       return false;
 566
 567    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 568    DF->File = CurrentFile - Cache.PkgFileP;
 569
 570    // Link it to the end of the list
 571    map_ptrloc *Last = &Desc->FileList;
 572    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 573       Last = &D->NextFile;
 574
 575    DF->NextFile = *Last;
 576    *Last = DF.Index();
 577
 578    DF->Offset = List.Offset();
 579    DF->Size = List.Size();
 580    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 581       Cache.HeaderP->MaxDescFileSize = DF->Size;
 582    Cache.HeaderP->DescFileCount++;
 583
 584    return true;
 585 }
 586                                                                         /*}}}*/
 587 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 588 // ---------------------------------------------------------------------
 589 /* This puts a description structure in the linked list */
 590 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 591                                             const string &Lang,
 592                                             const MD5SumValue &md5sum,
 593                                             map_ptrloc Next)
 594 {
 595    // Get a structure
 596    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 597    if (Description == 0)
 598       return 0;
 599
 600    // Fill it in
 601    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 602    Desc->NextDesc = Next;
 603    Desc->ID = Cache.HeaderP->DescriptionCount++;
 604    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 605    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 606    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 607       return 0;
 608    Desc->language_code = idxlanguage_code;
 609    Desc->md5sum = idxmd5sum;
 610
 611    return Description;
 612 }
 613                                                                         /*}}}*/
 614 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 615 // ---------------------------------------------------------------------
 616 /* This prepares the Cache for delivery */
 617 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 618 {
 619    // FIXME: add progress reporting for this operation
 620    // Do we have different architectures in your groups ?
 621    vector<string> archs = APT::Configuration::getArchitectures();
 622    if (archs.size() > 1)
 623    {
 624       // Create Conflicts in between the group
 625       pkgCache::GrpIterator G = GetCache().GrpBegin();
 626       Dynamic<pkgCache::GrpIterator> DynG(G);
 627       for (; G.end() != true; G++)
 628       {
 629          string const PkgName = G.Name();
 630          pkgCache::PkgIterator P = G.PackageList();
 631          Dynamic<pkgCache::PkgIterator> DynP(P);
 632          for (; P.end() != true; P = G.NextPkg(P))
 633          {
 634             pkgCache::PkgIterator allPkg;
 635             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 636             pkgCache::VerIterator V = P.VersionList();
 637             Dynamic<pkgCache::VerIterator> DynV(V);
 638             for (; V.end() != true; V++)
 639             {
 640                char const * const Arch = P.Arch();
 641                map_ptrloc *OldDepLast = NULL;
 642                /* MultiArch handling introduces a lot of implicit Dependencies:
 643                 - MultiArch: same → Co-Installable if they have the same version
 644                 - Architecture: all → Need to be Co-Installable for internal reasons
 645                 - All others conflict with all other group members */
 646                bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 647                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 648                {
 649                   if (Arch == 0 || *A == Arch)
 650                      continue;
 651                   /* We allow only one installed arch at the time
 652                      per group, therefore each group member conflicts
 653                      with all other group members */
 654                   pkgCache::PkgIterator D = G.FindPkg(*A);
 655                   Dynamic<pkgCache::PkgIterator> DynD(D);
 656                   if (D.end() == true)
 657                      continue;
 658                   if (coInstall == true)
 659                   {
 660                      // Replaces: ${self}:other ( << ${binary:Version})
 661                      NewDepends(D, V, V.VerStr(),
 662                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 663                                 OldDepLast);
 664                      // Breaks: ${self}:other (!= ${binary:Version})
 665                      NewDepends(D, V, V.VerStr(),
 666                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 667                                 OldDepLast);
 668                   } else {
 669                         // Conflicts: ${self}:other
 670                         NewDepends(D, V, "",
 671                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 672                                    OldDepLast);
 673                   }
 674                }
 675             }
 676          }
 677       }
 678    }
 679    return true;
 680 }
 681                                                                         /*}}}*/
 682 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 683 // ---------------------------------------------------------------------
 684 /* This creates a dependency element in the tree. It is linked to the
 685    version and to the package that it is pointing to. */
 686 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 687                                    pkgCache::VerIterator &Ver,
 688                                    string const &Version,
 689                                    unsigned int const &Op,
 690                                    unsigned int const &Type,
 691                                    map_ptrloc* &OldDepLast)
 692 {
 693    void const * const oldMap = Map.Data();
 694    // Get a structure
 695    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 696    if (unlikely(Dependency == 0))
 697       return false;
 698
 699    // Fill it in
 700    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 701    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 702    Dep->ParentVer = Ver.Index();
 703    Dep->Type = Type;
 704    Dep->CompareOp = Op;
 705    Dep->ID = Cache.HeaderP->DependsCount++;
 706
 707    // Probe the reverse dependency list for a version string that matches
 708    if (Version.empty() == false)
 709    {
 710 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 711          if (I->Version != 0 && I.TargetVer() == Version)
 712             Dep->Version = I->Version;*/
 713       if (Dep->Version == 0) {
 714          map_ptrloc const index = WriteStringInMap(Version);
 715          if (unlikely(index == 0))
 716             return false;
 717          Dep->Version = index;
 718       }
 719    }
 720
 721    // Link it to the package
 722    Dep->Package = Pkg.Index();
 723    Dep->NextRevDepends = Pkg->RevDepends;
 724    Pkg->RevDepends = Dep.Index();
 725
 726    // Do we know where to link the Dependency to?
 727    if (OldDepLast == NULL)
 728    {
 729       OldDepLast = &Ver->DependsList;
 730       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 731          OldDepLast = &D->NextDepends;
 732    } else if (oldMap != Map.Data())
 733       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 734
 735    Dep->NextDepends = *OldDepLast;
 736    *OldDepLast = Dep.Index();
 737    OldDepLast = &Dep->NextDepends;
 738
 739    return true;
 740 }
 741                                                                         /*}}}*/
 742 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 743 // ---------------------------------------------------------------------
 744 /* This creates a Group and the Package to link this dependency to if
 745    needed and handles also the caching of the old endpoint */
 746 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 747                                                const string &PackageName,
 748                                                const string &Arch,
 749                                                const string &Version,
 750                                                unsigned int Op,
 751                                                unsigned int Type)
 752 {
 753    pkgCache::GrpIterator Grp;
 754    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 755    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 756       return false;
 757
 758    // Locate the target package
 759    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 760    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 761    if (Pkg.end() == true) {
 762       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 763          return false;
 764    }
 765
 766    // Is it a file dependency?
 767    if (unlikely(PackageName[0] == '/'))
 768       FoundFileDeps = true;
 769
 770    /* Caching the old end point speeds up generation substantially */
 771    if (OldDepVer != Ver) {
 772       OldDepLast = NULL;
 773       OldDepVer = Ver;
 774    }
 775
 776    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 777 }
 778                                                                         /*}}}*/
 779 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 780 // ---------------------------------------------------------------------
 781 /* */
 782 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 783                                                 const string &PkgName,
 784                                                 const string &PkgArch,
 785                                                 const string &Version)
 786 {
 787    pkgCache &Cache = Owner->Cache;
 788
 789    // We do not add self referencing provides
 790    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 791         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 792       return true;
 793
 794    // Get a structure
 795    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 796    if (unlikely(Provides == 0))
 797       return false;
 798    Cache.HeaderP->ProvidesCount++;
 799
 800    // Fill it in
 801    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 802    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 803    Prv->Version = Ver.Index();
 804    Prv->NextPkgProv = Ver->ProvidesList;
 805    Ver->ProvidesList = Prv.Index();
 806    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 807       return false;
 808
 809    // Locate the target package
 810    pkgCache::PkgIterator Pkg;
 811    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 812    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 813       return false;
 814
 815    // Link it to the package
 816    Prv->ParentPkg = Pkg.Index();
 817    Prv->NextProvides = Pkg->ProvidesList;
 818    Pkg->ProvidesList = Prv.Index();
 819
 820    return true;
 821 }
 822                                                                         /*}}}*/
 823 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 824 // ---------------------------------------------------------------------
 825 /* This is used to select which file is to be associated with all newly
 826    added versions. The caller is responsible for setting the IMS fields. */
 827 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 828                                    const pkgIndexFile &Index,
 829                                    unsigned long Flags)
 830 {
 831    // Get some space for the structure
 832    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 833    if (unlikely(idxFile == 0))
 834       return false;
 835    CurrentFile = Cache.PkgFileP + idxFile;
 836
 837    // Fill it in
 838    map_ptrloc const idxFileName = WriteStringInMap(File);
 839    map_ptrloc const idxSite = WriteUniqString(Site);
 840    if (unlikely(idxFileName == 0 || idxSite == 0))
 841       return false;
 842    CurrentFile->FileName = idxFileName;
 843    CurrentFile->Site = idxSite;
 844    CurrentFile->NextFile = Cache.HeaderP->FileList;
 845    CurrentFile->Flags = Flags;
 846    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 847    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 848    if (unlikely(idxIndexType == 0))
 849       return false;
 850    CurrentFile->IndexType = idxIndexType;
 851    PkgFileName = File;
 852    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 853    Cache.HeaderP->PackageFileCount++;
 854
 855    if (Progress != 0)
 856       Progress->SubProgress(Index.Size());
 857    return true;
 858 }
 859                                                                         /*}}}*/
 860 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 861 // ---------------------------------------------------------------------
 862 /* This is used to create handles to strings. Given the same text it
 863    always returns the same number */
 864 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 865                                                  unsigned int Size)
 866 {
 867    /* We use a very small transient hash table here, this speeds up generation
 868       by a fair amount on slower machines */
 869    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 870    if (Bucket != 0 &&
 871        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 872       return Bucket->String;
 873
 874    // Search for an insertion point
 875    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 876    int Res = 1;
 877    map_ptrloc *Last = &Cache.HeaderP->StringList;
 878    for (; I != Cache.StringItemP; Last = &I->NextItem,
 879         I = Cache.StringItemP + I->NextItem)
 880    {
 881       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 882       if (Res >= 0)
 883          break;
 884    }
 885
 886    // Match
 887    if (Res == 0)
 888    {
 889       Bucket = I;
 890       return I->String;
 891    }
 892
 893    // Get a structure
 894    void const * const oldMap = Map.Data();
 895    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 896    if (Item == 0)
 897       return 0;
 898
 899    map_ptrloc const idxString = WriteStringInMap(S,Size);
 900    if (unlikely(idxString == 0))
 901       return 0;
 902    if (oldMap != Map.Data()) {
 903       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 904       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 905    }
 906    *Last = Item;
 907
 908    // Fill in the structure
 909    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 910    ItemP->NextItem = I - Cache.StringItemP;
 911    ItemP->String = idxString;
 912
 913    Bucket = ItemP;
 914    return ItemP->String;
 915 }
 916                                                                         /*}}}*/
 917 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 918 // ---------------------------------------------------------------------
 919 /* This just verifies that each file in the list of index files exists,
 920    has matching attributes with the cache and the cache does not have
 921    any extra files. */
 922 static bool CheckValidity(const string &CacheFile,
 923                           pkgSourceList &List,
 924                           FileIterator Start,
 925                           FileIterator End,
 926                           MMap **OutMap = 0)
 927 {
 928    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 929    // No file, certainly invalid
 930    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 931    {
 932       if (Debug == true)
 933          std::clog << "CacheFile doesn't exist" << std::endl;
 934       return false;
 935    }
 936
 937    if (List.GetLastModifiedTime() < GetModificationTime(CacheFile))
 938    {
 939       if (Debug == true)
 940          std::clog << "sources.list is newer than the cache" << std::endl;
 941       return false;
 942    }
 943
 944    // Map it
 945    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 946    SPtr<MMap> Map = new MMap(CacheF,0);
 947    pkgCache Cache(Map);
 948    if (_error->PendingError() == true || Map->Size() == 0)
 949    {
 950       if (Debug == true)
 951          std::clog << "Errors are pending or Map is empty()" << std::endl;
 952       _error->Discard();
 953       return false;
 954    }
 955
 956    /* Now we check every index file, see if it is in the cache,
 957       verify the IMS data and check that it is on the disk too.. */
 958    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 959    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 960    for (; Start != End; Start++)
 961    {
 962       if (Debug == true)
 963          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 964       if ((*Start)->HasPackages() == false)
 965       {
 966          if (Debug == true)
 967             std::clog << "Has NO packages" << std::endl;
 968          continue;
 969       }
 970
 971       if ((*Start)->Exists() == false)
 972       {
 973 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 974          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 975                           (*Start)->Describe().c_str());
 976 #endif
 977          if (Debug == true)
 978             std::clog << "file doesn't exist" << std::endl;
 979          continue;
 980       }
 981
 982       // FindInCache is also expected to do an IMS check.
 983       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 984       if (File.end() == true)
 985       {
 986          if (Debug == true)
 987             std::clog << "FindInCache returned end-Pointer" << std::endl;
 988          return false;
 989       }
 990
 991       Visited[File->ID] = true;
 992       if (Debug == true)
 993          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 994    }
 995
 996    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 997       if (Visited[I] == false)
 998       {
 999          if (Debug == true)
1000             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1001          return false;
1002       }
1003
1004    if (_error->PendingError() == true)
1005    {
1006       if (Debug == true)
1007       {
1008          std::clog << "Validity failed because of pending errors:" << std::endl;
1009          _error->DumpErrors();
1010       }
1011       _error->Discard();
1012       return false;
1013    }
1014
1015    if (OutMap != 0)
1016       *OutMap = Map.UnGuard();
1017    return true;
1018 }
1019                                                                         /*}}}*/
1020 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1021 // ---------------------------------------------------------------------
1022 /* Size is kind of an abstract notion that is only used for the progress
1023    meter */
1024 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1025 {
1026    unsigned long TotalSize = 0;
1027    for (; Start != End; Start++)
1028    {
1029       if ((*Start)->HasPackages() == false)
1030          continue;
1031       TotalSize += (*Start)->Size();
1032    }
1033    return TotalSize;
1034 }
1035                                                                         /*}}}*/
1036 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1037 // ---------------------------------------------------------------------
1038 /* */
1039 static bool BuildCache(pkgCacheGenerator &Gen,
1040                        OpProgress *Progress,
1041                        unsigned long &CurrentSize,unsigned long TotalSize,
1042                        FileIterator Start, FileIterator End)
1043 {
1044    FileIterator I;
1045    for (I = Start; I != End; I++)
1046    {
1047       if ((*I)->HasPackages() == false)
1048          continue;
1049
1050       if ((*I)->Exists() == false)
1051          continue;
1052
1053       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1054       {
1055          _error->Warning("Duplicate sources.list entry %s",
1056                          (*I)->Describe().c_str());
1057          continue;
1058       }
1059
1060       unsigned long Size = (*I)->Size();
1061       if (Progress != NULL)
1062          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1063       CurrentSize += Size;
1064
1065       if ((*I)->Merge(Gen,Progress) == false)
1066          return false;
1067    }
1068
1069    if (Gen.HasFileDeps() == true)
1070    {
1071       if (Progress != NULL)
1072          Progress->Done();
1073       TotalSize = ComputeSize(Start, End);
1074       CurrentSize = 0;
1075       for (I = Start; I != End; I++)
1076       {
1077          unsigned long Size = (*I)->Size();
1078          if (Progress != NULL)
1079             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1080          CurrentSize += Size;
1081          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1082             return false;
1083       }
1084    }
1085
1086    return true;
1087 }
1088                                                                         /*}}}*/
1089 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1090 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1091    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1092    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1093    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1094    Flags |= MMap::Moveable;
1095    if (_config->FindB("APT::Cache-Fallback", false) == true)
1096       Flags |= MMap::Fallback;
1097    if (CacheF != NULL)
1098       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1099    else
1100       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1101 }
1102                                                                         /*}}}*/
1103 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1104 // ---------------------------------------------------------------------
1105 /* This makes sure that the status cache (the cache that has all
1106    index files from the sources list and all local ones) is ready
1107    to be mmaped. If OutMap is not zero then a MMap object representing
1108    the cache will be stored there. This is pretty much mandetory if you
1109    are using AllowMem. AllowMem lets the function be run as non-root
1110    where it builds the cache 'fast' into a memory buffer. */
1111 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1112                         MMap **OutMap, bool AllowMem)
1113    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1114 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1115                         MMap **OutMap,bool AllowMem)
1116 {
1117    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1118
1119    vector<pkgIndexFile *> Files;
1120    for (vector<metaIndex *>::const_iterator i = List.begin();
1121         i != List.end();
1122         i++)
1123    {
1124       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1125       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1126            j != Indexes->end();
1127            j++)
1128          Files.push_back (*j);
1129    }
1130
1131    unsigned long const EndOfSource = Files.size();
1132    if (_system->AddStatusFiles(Files) == false)
1133       return false;
1134
1135    // Decide if we can write to the files..
1136    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1137    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1138
1139    // ensure the cache directory exists
1140    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1141    {
1142       string dir = _config->FindDir("Dir::Cache");
1143       size_t const len = dir.size();
1144       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1145          dir = dir.substr(0, len - 5);
1146       if (CacheFile.empty() == false)
1147          CreateDirectory(dir, flNotFile(CacheFile));
1148       if (SrcCacheFile.empty() == false)
1149          CreateDirectory(dir, flNotFile(SrcCacheFile));
1150    }
1151
1152    // Decide if we can write to the cache
1153    bool Writeable = false;
1154    if (CacheFile.empty() == false)
1155       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1156    else
1157       if (SrcCacheFile.empty() == false)
1158          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1159    if (Debug == true)
1160       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1161
1162    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1163       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1164
1165    if (Progress != NULL)
1166       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1167
1168    // Cache is OK, Fin.
1169    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1170    {
1171       if (Progress != NULL)
1172          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1173       if (Debug == true)
1174          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1175       return true;
1176    }
1177    else if (Debug == true)
1178          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1179
1180    /* At this point we know we need to reconstruct the package cache,
1181       begin. */
1182    SPtr<FileFd> CacheF;
1183    SPtr<DynamicMMap> Map;
1184    if (Writeable == true && CacheFile.empty() == false)
1185    {
1186       _error->PushToStack();
1187       unlink(CacheFile.c_str());
1188       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1189       fchmod(CacheF->Fd(),0644);
1190       Map = CreateDynamicMMap(CacheF, MMap::Public);
1191       if (_error->PendingError() == true)
1192       {
1193          delete CacheF.UnGuard();
1194          delete Map.UnGuard();
1195          if (Debug == true)
1196             std::clog << "Open filebased MMap FAILED" << std::endl;
1197          Writeable = false;
1198          if (AllowMem == false)
1199          {
1200             _error->MergeWithStack();
1201             return false;
1202          }
1203          _error->RevertToStack();
1204       }
1205       else if (Debug == true)
1206       {
1207          _error->MergeWithStack();
1208          std::clog << "Open filebased MMap" << std::endl;
1209       }
1210    }
1211    if (Writeable == false || CacheFile.empty() == true)
1212    {
1213       // Just build it in memory..
1214       Map = CreateDynamicMMap(NULL);
1215       if (Debug == true)
1216          std::clog << "Open memory Map (not filebased)" << std::endl;
1217    }
1218
1219    // Lets try the source cache.
1220    unsigned long CurrentSize = 0;
1221    unsigned long TotalSize = 0;
1222    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1223                      Files.begin()+EndOfSource) == true)
1224    {
1225       if (Debug == true)
1226          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1227       // Preload the map with the source cache
1228       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1229       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1230       if ((alloc == 0 && _error->PendingError())
1231                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1232                                 SCacheF.Size()) == false)
1233          return false;
1234
1235       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1236
1237       // Build the status cache
1238       pkgCacheGenerator Gen(Map.Get(),Progress);
1239       if (_error->PendingError() == true)
1240          return false;
1241       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1242                      Files.begin()+EndOfSource,Files.end()) == false)
1243          return false;
1244
1245       // FIXME: move me to a better place
1246       Gen.FinishCache(Progress);
1247    }
1248    else
1249    {
1250       if (Debug == true)
1251          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1252       TotalSize = ComputeSize(Files.begin(),Files.end());
1253
1254       // Build the source cache
1255       pkgCacheGenerator Gen(Map.Get(),Progress);
1256       if (_error->PendingError() == true)
1257          return false;
1258       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1259                      Files.begin(),Files.begin()+EndOfSource) == false)
1260          return false;
1261
1262       // Write it back
1263       if (Writeable == true && SrcCacheFile.empty() == false)
1264       {
1265          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1266          if (_error->PendingError() == true)
1267             return false;
1268
1269          fchmod(SCacheF.Fd(),0644);
1270
1271          // Write out the main data
1272          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1273             return _error->Error(_("IO Error saving source cache"));
1274          SCacheF.Sync();
1275
1276          // Write out the proper header
1277          Gen.GetCache().HeaderP->Dirty = false;
1278          if (SCacheF.Seek(0) == false ||
1279              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1280             return _error->Error(_("IO Error saving source cache"));
1281          Gen.GetCache().HeaderP->Dirty = true;
1282          SCacheF.Sync();
1283       }
1284
1285       // Build the status cache
1286       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1287                      Files.begin()+EndOfSource,Files.end()) == false)
1288          return false;
1289
1290       // FIXME: move me to a better place
1291       Gen.FinishCache(Progress);
1292    }
1293    if (Debug == true)
1294       std::clog << "Caches are ready for shipping" << std::endl;
1295
1296    if (_error->PendingError() == true)
1297       return false;
1298    if (OutMap != 0)
1299    {
1300       if (CacheF != 0)
1301       {
1302          delete Map.UnGuard();
1303          *OutMap = new MMap(*CacheF,0);
1304       }
1305       else
1306       {
1307          *OutMap = Map.UnGuard();
1308       }
1309    }
1310
1311    return true;
1312 }
1313                                                                         /*}}}*/
1314 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1315 // ---------------------------------------------------------------------
1316 /* */
1317 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1318    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1319 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1320 {
1321    vector<pkgIndexFile *> Files;
1322    unsigned long EndOfSource = Files.size();
1323    if (_system->AddStatusFiles(Files) == false)
1324       return false;
1325
1326    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1327    unsigned long CurrentSize = 0;
1328    unsigned long TotalSize = 0;
1329
1330    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1331
1332    // Build the status cache
1333    if (Progress != NULL)
1334       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1335    pkgCacheGenerator Gen(Map.Get(),Progress);
1336    if (_error->PendingError() == true)
1337       return false;
1338    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1339                   Files.begin()+EndOfSource,Files.end()) == false)
1340       return false;
1341
1342    // FIXME: move me to a better place
1343    Gen.FinishCache(Progress);
1344
1345    if (_error->PendingError() == true)
1346       return false;
1347    *OutMap = Map.UnGuard();
1348
1349    return true;
1350 }
1351                                                                         /*}}}*/