apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26
  27 #include <apt-pkg/tagfile.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Cache.HeaderP->CacheFileSize = Map.Size();
 102    Map.Sync(0,sizeof(pkgCache::Header));
 103 }
 104                                                                         /*}}}*/
 105 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 106    if (oldMap == newMap)
 107       return;
 108
 109    Cache.ReMap(false);
 110
 111    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 112
 113    for (size_t i = 0; i < _count(UniqHash); ++i)
 114       if (UniqHash[i] != 0)
 115          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 116
 117    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 118         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 119       (*i)->ReMap(oldMap, newMap);
 120    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 121         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 122       (*i)->ReMap(oldMap, newMap);
 123    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 124         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 125       (*i)->ReMap(oldMap, newMap);
 126    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 127         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 128       (*i)->ReMap(oldMap, newMap);
 129    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 130         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 131       (*i)->ReMap(oldMap, newMap);
 132    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 133         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 134       (*i)->ReMap(oldMap, newMap);
 135    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 136         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 137       (*i)->ReMap(oldMap, newMap);
 138 }                                                                       /*}}}*/
 139 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 140 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 141                                         const unsigned long &Len) {
 142    void const * const oldMap = Map.Data();
 143    map_ptrloc const index = Map.WriteString(String, Len);
 144    if (index != 0)
 145       ReMap(oldMap, Map.Data());
 146    return index;
 147 }
 148                                                                         /*}}}*/
 149 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 150 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 151    void const * const oldMap = Map.Data();
 152    map_ptrloc const index = Map.WriteString(String);
 153    if (index != 0)
 154       ReMap(oldMap, Map.Data());
 155    return index;
 156 }
 157                                                                         /*}}}*/
 158 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 159    void const * const oldMap = Map.Data();
 160    map_ptrloc const index = Map.Allocate(size);
 161    if (index != 0)
 162       ReMap(oldMap, Map.Data());
 163    return index;
 164 }
 165                                                                         /*}}}*/
 166 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 167 // ---------------------------------------------------------------------
 168 /* This provides the generation of the entries in the cache. Each loop
 169    goes through a single package record from the underlying parse engine. */
 170 bool pkgCacheGenerator::MergeList(ListParser &List,
 171                                   pkgCache::VerIterator *OutVer)
 172 {
 173    List.Owner = this;
 174
 175    unsigned int Counter = 0;
 176    while (List.Step() == true)
 177    {
 178       string const PackageName = List.Package();
 179       if (PackageName.empty() == true)
 180          return false;
 181
 182       string const Arch = List.Architecture();
 183
 184       // Get a pointer to the package structure
 185       pkgCache::PkgIterator Pkg;
 186       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 187       if (NewPackage(Pkg, PackageName, Arch) == false)
 188          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 189       Counter++;
 190       if (Counter % 100 == 0 && Progress != 0)
 191          Progress->Progress(List.Offset());
 192
 193       /* Get a pointer to the version structure. We know the list is sorted
 194          so we use that fact in the search. Insertion of new versions is
 195          done with correct sorting */
 196       string Version = List.Version();
 197       if (Version.empty() == true)
 198       {
 199          // we first process the package, then the descriptions
 200          // (this has the bonus that we get MMap error when we run out
 201          //  of MMap space)
 202          pkgCache::VerIterator Ver(Cache);
 203          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 204          if (List.UsePackage(Pkg, Ver) == false)
 205             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 206                                  PackageName.c_str());
 207
 208          // Find the right version to write the description
 209          MD5SumValue CurMd5 = List.Description_md5();
 210          Ver = Pkg.VersionList();
 211
 212          for (; Ver.end() == false; ++Ver)
 213          {
 214             pkgCache::DescIterator Desc = Ver.DescriptionList();
 215             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 216             map_ptrloc *LastDesc = &Ver->DescriptionList;
 217             bool duplicate=false;
 218
 219             // don't add a new description if we have one for the given
 220             // md5 && language
 221             for ( ; Desc.end() == false; Desc++)
 222                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 223                    Desc.LanguageCode() == List.DescriptionLanguage())
 224                   duplicate=true;
 225             if(duplicate)
 226                continue;
 227
 228             for (Desc = Ver.DescriptionList();
 229                  Desc.end() == false;
 230                  LastDesc = &Desc->NextDesc, Desc++)
 231             {
 232                if (MD5SumValue(Desc.md5()) == CurMd5)
 233                {
 234                   // Add new description
 235                   void const * const oldMap = Map.Data();
 236                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 237                   if (oldMap != Map.Data())
 238                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 239                   *LastDesc = descindex;
 240                   Desc->ParentPkg = Pkg.Index();
 241
 242                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 243                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 244                   break;
 245                }
 246             }
 247          }
 248
 249          continue;
 250       }
 251
 252       pkgCache::VerIterator Ver = Pkg.VersionList();
 253       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 254       map_ptrloc *LastVer = &Pkg->VersionList;
 255       void const * oldMap = Map.Data();
 256       int Res = 1;
 257       unsigned long const Hash = List.VersionHash();
 258       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 259       {
 260          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 261          // Version is higher as current version - insert here
 262          if (Res > 0)
 263             break;
 264          // Versionstrings are equal - is hash also equal?
 265          if (Res == 0 && Ver->Hash == Hash)
 266             break;
 267          // proceed with the next till we have either the right
 268          // or we found another version (which will be lower)
 269       }
 270
 271       /* We already have a version for this item, record that we saw it */
 272       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 273       {
 274          if (List.UsePackage(Pkg,Ver) == false)
 275             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 276                                  PackageName.c_str());
 277
 278          if (NewFileVer(Ver,List) == false)
 279             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 280                                  PackageName.c_str());
 281
 282          // Read only a single record and return
 283          if (OutVer != 0)
 284          {
 285             *OutVer = Ver;
 286             FoundFileDeps |= List.HasFileDeps();
 287             return true;
 288          }
 289
 290          continue;
 291       }
 292
 293       // Add a new version
 294       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 295       if (verindex == 0 && _error->PendingError())
 296          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 297                               PackageName.c_str(), 1);
 298
 299       if (oldMap != Map.Data())
 300          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 301       *LastVer = verindex;
 302       Ver->ParentPkg = Pkg.Index();
 303       Ver->Hash = Hash;
 304
 305       if (List.NewVersion(Ver) == false)
 306          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 307                               PackageName.c_str(), 2);
 308
 309       if (List.UsePackage(Pkg,Ver) == false)
 310          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 311                               PackageName.c_str());
 312
 313       if (NewFileVer(Ver,List) == false)
 314          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 315                               PackageName.c_str(), 3);
 316
 317       // Read only a single record and return
 318       if (OutVer != 0)
 319       {
 320          *OutVer = Ver;
 321          FoundFileDeps |= List.HasFileDeps();
 322          return true;
 323       }
 324
 325       /* Record the Description data. Description data always exist in
 326          Packages and Translation-* files. */
 327       pkgCache::DescIterator Desc = Ver.DescriptionList();
 328       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 329       map_ptrloc *LastDesc = &Ver->DescriptionList;
 330
 331       // Skip to the end of description set
 332       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 333
 334       // Add new description
 335       oldMap = Map.Data();
 336       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 337       if (oldMap != Map.Data())
 338          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 339       *LastDesc = descindex;
 340       Desc->ParentPkg = Pkg.Index();
 341
 342       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 343          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 344    }
 345
 346    FoundFileDeps |= List.HasFileDeps();
 347
 348    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 349       return _error->Error(_("Wow, you exceeded the number of package "
 350                              "names this APT is capable of."));
 351    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 352       return _error->Error(_("Wow, you exceeded the number of versions "
 353                              "this APT is capable of."));
 354    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 355       return _error->Error(_("Wow, you exceeded the number of descriptions "
 356                              "this APT is capable of."));
 357    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 358       return _error->Error(_("Wow, you exceeded the number of dependencies "
 359                              "this APT is capable of."));
 360    return true;
 361 }
 362                                                                         /*}}}*/
 363 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 364 // ---------------------------------------------------------------------
 365 /* If we found any file depends while parsing the main list we need to
 366    resolve them. Since it is undesired to load the entire list of files
 367    into the cache as virtual packages we do a two stage effort. MergeList
 368    identifies the file depends and this creates Provdies for them by
 369    re-parsing all the indexs. */
 370 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 371 {
 372    List.Owner = this;
 373
 374    unsigned int Counter = 0;
 375    while (List.Step() == true)
 376    {
 377       string PackageName = List.Package();
 378       if (PackageName.empty() == true)
 379          return false;
 380       string Version = List.Version();
 381       if (Version.empty() == true)
 382          continue;
 383
 384       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 385       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 386       if (Pkg.end() == true)
 387          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 388                                 PackageName.c_str());
 389       Counter++;
 390       if (Counter % 100 == 0 && Progress != 0)
 391          Progress->Progress(List.Offset());
 392
 393       unsigned long Hash = List.VersionHash();
 394       pkgCache::VerIterator Ver = Pkg.VersionList();
 395       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 396       for (; Ver.end() == false; Ver++)
 397       {
 398          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 399          {
 400             if (List.CollectFileProvides(Cache,Ver) == false)
 401                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 402             break;
 403          }
 404       }
 405
 406       if (Ver.end() == true)
 407          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 408    }
 409
 410    return true;
 411 }
 412                                                                         /*}}}*/
 413 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 414 // ---------------------------------------------------------------------
 415 /* This creates a new group structure and adds it to the hash table */
 416 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 417 {
 418    Grp = Cache.FindGrp(Name);
 419    if (Grp.end() == false)
 420       return true;
 421
 422    // Get a structure
 423    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 424    if (unlikely(Group == 0))
 425       return false;
 426
 427    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 428    map_ptrloc const idxName = WriteStringInMap(Name);
 429    if (unlikely(idxName == 0))
 430       return false;
 431    Grp->Name = idxName;
 432
 433    // Insert it into the hash table
 434    unsigned long const Hash = Cache.Hash(Name);
 435    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 436    Cache.HeaderP->GrpHashTable[Hash] = Group;
 437
 438    Grp->ID = Cache.HeaderP->GroupCount++;
 439    return true;
 440 }
 441                                                                         /*}}}*/
 442 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 443 // ---------------------------------------------------------------------
 444 /* This creates a new package structure and adds it to the hash table */
 445 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 446                                         const string &Arch) {
 447    pkgCache::GrpIterator Grp;
 448    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 449    if (unlikely(NewGroup(Grp, Name) == false))
 450       return false;
 451
 452    Pkg = Grp.FindPkg(Arch);
 453       if (Pkg.end() == false)
 454          return true;
 455
 456    // Get a structure
 457    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 458    if (unlikely(Package == 0))
 459       return false;
 460    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 461
 462    // Insert the package into our package list
 463    if (Grp->FirstPackage == 0) // the group is new
 464    {
 465       // Insert it into the hash table
 466       unsigned long const Hash = Cache.Hash(Name);
 467       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 468       Cache.HeaderP->PkgHashTable[Hash] = Package;
 469       Grp->FirstPackage = Package;
 470    }
 471    else // Group the Packages together
 472    {
 473       // this package is the new last package
 474       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 475       Pkg->NextPackage = LastPkg->NextPackage;
 476       LastPkg->NextPackage = Package;
 477    }
 478    Grp->LastPackage = Package;
 479
 480    // Set the name, arch and the ID
 481    Pkg->Name = Grp->Name;
 482    Pkg->Group = Grp.Index();
 483    // all is mapped to the native architecture
 484    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 485    if (unlikely(idxArch == 0))
 486       return false;
 487    Pkg->Arch = idxArch;
 488    Pkg->ID = Cache.HeaderP->PackageCount++;
 489
 490    return true;
 491 }
 492                                                                         /*}}}*/
 493 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 494 // ---------------------------------------------------------------------
 495 /* */
 496 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 497                                    ListParser &List)
 498 {
 499    if (CurrentFile == 0)
 500       return true;
 501
 502    // Get a structure
 503    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 504    if (VerFile == 0)
 505       return 0;
 506
 507    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 508    VF->File = CurrentFile - Cache.PkgFileP;
 509
 510    // Link it to the end of the list
 511    map_ptrloc *Last = &Ver->FileList;
 512    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 513       Last = &V->NextFile;
 514    VF->NextFile = *Last;
 515    *Last = VF.Index();
 516
 517    VF->Offset = List.Offset();
 518    VF->Size = List.Size();
 519    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 520       Cache.HeaderP->MaxVerFileSize = VF->Size;
 521    Cache.HeaderP->VerFileCount++;
 522
 523    return true;
 524 }
 525                                                                         /*}}}*/
 526 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 527 // ---------------------------------------------------------------------
 528 /* This puts a version structure in the linked list */
 529 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 530                                             const string &VerStr,
 531                                             unsigned long Next)
 532 {
 533    // Get a structure
 534    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 535    if (Version == 0)
 536       return 0;
 537
 538    // Fill it in
 539    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 540    Ver->NextVer = Next;
 541    Ver->ID = Cache.HeaderP->VersionCount++;
 542    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 543    if (unlikely(idxVerStr == 0))
 544       return 0;
 545    Ver->VerStr = idxVerStr;
 546
 547    return Version;
 548 }
 549                                                                         /*}}}*/
 550 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 551 // ---------------------------------------------------------------------
 552 /* */
 553 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 554                                    ListParser &List)
 555 {
 556    if (CurrentFile == 0)
 557       return true;
 558
 559    // Get a structure
 560    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 561    if (DescFile == 0)
 562       return false;
 563
 564    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 565    DF->File = CurrentFile - Cache.PkgFileP;
 566
 567    // Link it to the end of the list
 568    map_ptrloc *Last = &Desc->FileList;
 569    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 570       Last = &D->NextFile;
 571
 572    DF->NextFile = *Last;
 573    *Last = DF.Index();
 574
 575    DF->Offset = List.Offset();
 576    DF->Size = List.Size();
 577    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 578       Cache.HeaderP->MaxDescFileSize = DF->Size;
 579    Cache.HeaderP->DescFileCount++;
 580
 581    return true;
 582 }
 583                                                                         /*}}}*/
 584 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 585 // ---------------------------------------------------------------------
 586 /* This puts a description structure in the linked list */
 587 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 588                                             const string &Lang,
 589                                             const MD5SumValue &md5sum,
 590                                             map_ptrloc Next)
 591 {
 592    // Get a structure
 593    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 594    if (Description == 0)
 595       return 0;
 596
 597    // Fill it in
 598    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 599    Desc->NextDesc = Next;
 600    Desc->ID = Cache.HeaderP->DescriptionCount++;
 601    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 602    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 603    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 604       return 0;
 605    Desc->language_code = idxlanguage_code;
 606    Desc->md5sum = idxmd5sum;
 607
 608    return Description;
 609 }
 610                                                                         /*}}}*/
 611 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 612 // ---------------------------------------------------------------------
 613 /* This prepares the Cache for delivery */
 614 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 615 {
 616    // FIXME: add progress reporting for this operation
 617    // Do we have different architectures in your groups ?
 618    vector<string> archs = APT::Configuration::getArchitectures();
 619    if (archs.size() > 1)
 620    {
 621       // Create Conflicts in between the group
 622       pkgCache::GrpIterator G = GetCache().GrpBegin();
 623       Dynamic<pkgCache::GrpIterator> DynG(G);
 624       for (; G.end() != true; G++)
 625       {
 626          string const PkgName = G.Name();
 627          pkgCache::PkgIterator P = G.PackageList();
 628          Dynamic<pkgCache::PkgIterator> DynP(P);
 629          for (; P.end() != true; P = G.NextPkg(P))
 630          {
 631             pkgCache::PkgIterator allPkg;
 632             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 633             pkgCache::VerIterator V = P.VersionList();
 634             Dynamic<pkgCache::VerIterator> DynV(V);
 635             for (; V.end() != true; V++)
 636             {
 637                char const * const Arch = P.Arch();
 638                map_ptrloc *OldDepLast = NULL;
 639                /* MultiArch handling introduces a lot of implicit Dependencies:
 640                 - MultiArch: same → Co-Installable if they have the same version
 641                 - Architecture: all → Need to be Co-Installable for internal reasons
 642                 - All others conflict with all other group members */
 643                bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 644                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 645                {
 646                   if (Arch == 0 || *A == Arch)
 647                      continue;
 648                   /* We allow only one installed arch at the time
 649                      per group, therefore each group member conflicts
 650                      with all other group members */
 651                   pkgCache::PkgIterator D = G.FindPkg(*A);
 652                   Dynamic<pkgCache::PkgIterator> DynD(D);
 653                   if (D.end() == true)
 654                      continue;
 655                   if (coInstall == true)
 656                   {
 657                      // Replaces: ${self}:other ( << ${binary:Version})
 658                      NewDepends(D, V, V.VerStr(),
 659                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 660                                 OldDepLast);
 661                      // Breaks: ${self}:other (!= ${binary:Version})
 662                      NewDepends(D, V, V.VerStr(),
 663                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 664                                 OldDepLast);
 665                   } else {
 666                         // Conflicts: ${self}:other
 667                         NewDepends(D, V, "",
 668                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 669                                    OldDepLast);
 670                   }
 671                }
 672             }
 673          }
 674       }
 675    }
 676    return true;
 677 }
 678                                                                         /*}}}*/
 679 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 680 // ---------------------------------------------------------------------
 681 /* This creates a dependency element in the tree. It is linked to the
 682    version and to the package that it is pointing to. */
 683 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 684                                    pkgCache::VerIterator &Ver,
 685                                    string const &Version,
 686                                    unsigned int const &Op,
 687                                    unsigned int const &Type,
 688                                    map_ptrloc *OldDepLast)
 689 {
 690    void const * const oldMap = Map.Data();
 691    // Get a structure
 692    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 693    if (unlikely(Dependency == 0))
 694       return false;
 695
 696    // Fill it in
 697    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 698    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 699    Dep->ParentVer = Ver.Index();
 700    Dep->Type = Type;
 701    Dep->CompareOp = Op;
 702    Dep->ID = Cache.HeaderP->DependsCount++;
 703
 704    // Probe the reverse dependency list for a version string that matches
 705    if (Version.empty() == false)
 706    {
 707 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 708          if (I->Version != 0 && I.TargetVer() == Version)
 709             Dep->Version = I->Version;*/
 710       if (Dep->Version == 0) {
 711          map_ptrloc const index = WriteStringInMap(Version);
 712          if (unlikely(index == 0))
 713             return false;
 714          Dep->Version = index;
 715       }
 716    }
 717
 718    // Link it to the package
 719    Dep->Package = Pkg.Index();
 720    Dep->NextRevDepends = Pkg->RevDepends;
 721    Pkg->RevDepends = Dep.Index();
 722
 723    // Do we know where to link the Dependency to?
 724    if (OldDepLast == NULL)
 725    {
 726       OldDepLast = &Ver->DependsList;
 727       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 728          OldDepLast = &D->NextDepends;
 729    } else if (oldMap != Map.Data())
 730       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 731
 732    Dep->NextDepends = *OldDepLast;
 733    *OldDepLast = Dep.Index();
 734    OldDepLast = &Dep->NextDepends;
 735
 736    return true;
 737 }
 738                                                                         /*}}}*/
 739 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 740 // ---------------------------------------------------------------------
 741 /* This creates a Group and the Package to link this dependency to if
 742    needed and handles also the caching of the old endpoint */
 743 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 744                                                const string &PackageName,
 745                                                const string &Arch,
 746                                                const string &Version,
 747                                                unsigned int Op,
 748                                                unsigned int Type)
 749 {
 750    pkgCache::GrpIterator Grp;
 751    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 752    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 753       return false;
 754
 755    // Locate the target package
 756    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 757    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 758    if (Pkg.end() == true) {
 759       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 760          return false;
 761    }
 762
 763    // Is it a file dependency?
 764    if (unlikely(PackageName[0] == '/'))
 765       FoundFileDeps = true;
 766
 767    /* Caching the old end point speeds up generation substantially */
 768    if (OldDepVer != Ver) {
 769       OldDepLast = NULL;
 770       OldDepVer = Ver;
 771    }
 772
 773    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 774 }
 775                                                                         /*}}}*/
 776 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 777 // ---------------------------------------------------------------------
 778 /* */
 779 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 780                                                 const string &PkgName,
 781                                                 const string &PkgArch,
 782                                                 const string &Version)
 783 {
 784    pkgCache &Cache = Owner->Cache;
 785
 786    // We do not add self referencing provides
 787    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 788         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 789       return true;
 790
 791    // Get a structure
 792    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 793    if (unlikely(Provides == 0))
 794       return false;
 795    Cache.HeaderP->ProvidesCount++;
 796
 797    // Fill it in
 798    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 799    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 800    Prv->Version = Ver.Index();
 801    Prv->NextPkgProv = Ver->ProvidesList;
 802    Ver->ProvidesList = Prv.Index();
 803    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 804       return false;
 805
 806    // Locate the target package
 807    pkgCache::PkgIterator Pkg;
 808    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 809    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 810       return false;
 811
 812    // Link it to the package
 813    Prv->ParentPkg = Pkg.Index();
 814    Prv->NextProvides = Pkg->ProvidesList;
 815    Pkg->ProvidesList = Prv.Index();
 816
 817    return true;
 818 }
 819                                                                         /*}}}*/
 820 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 821 // ---------------------------------------------------------------------
 822 /* This is used to select which file is to be associated with all newly
 823    added versions. The caller is responsible for setting the IMS fields. */
 824 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 825                                    const pkgIndexFile &Index,
 826                                    unsigned long Flags)
 827 {
 828    // Get some space for the structure
 829    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 830    if (unlikely(idxFile == 0))
 831       return false;
 832    CurrentFile = Cache.PkgFileP + idxFile;
 833
 834    // Fill it in
 835    map_ptrloc const idxFileName = WriteStringInMap(File);
 836    map_ptrloc const idxSite = WriteUniqString(Site);
 837    if (unlikely(idxFileName == 0 || idxSite == 0))
 838       return false;
 839    CurrentFile->FileName = idxFileName;
 840    CurrentFile->Site = idxSite;
 841    CurrentFile->NextFile = Cache.HeaderP->FileList;
 842    CurrentFile->Flags = Flags;
 843    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 844    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 845    if (unlikely(idxIndexType == 0))
 846       return false;
 847    CurrentFile->IndexType = idxIndexType;
 848    PkgFileName = File;
 849    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 850    Cache.HeaderP->PackageFileCount++;
 851
 852    if (Progress != 0)
 853       Progress->SubProgress(Index.Size());
 854    return true;
 855 }
 856                                                                         /*}}}*/
 857 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 858 // ---------------------------------------------------------------------
 859 /* This is used to create handles to strings. Given the same text it
 860    always returns the same number */
 861 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 862                                                  unsigned int Size)
 863 {
 864    /* We use a very small transient hash table here, this speeds up generation
 865       by a fair amount on slower machines */
 866    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 867    if (Bucket != 0 &&
 868        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 869       return Bucket->String;
 870
 871    // Search for an insertion point
 872    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 873    int Res = 1;
 874    map_ptrloc *Last = &Cache.HeaderP->StringList;
 875    for (; I != Cache.StringItemP; Last = &I->NextItem,
 876         I = Cache.StringItemP + I->NextItem)
 877    {
 878       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 879       if (Res >= 0)
 880          break;
 881    }
 882
 883    // Match
 884    if (Res == 0)
 885    {
 886       Bucket = I;
 887       return I->String;
 888    }
 889
 890    // Get a structure
 891    void const * const oldMap = Map.Data();
 892    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 893    if (Item == 0)
 894       return 0;
 895
 896    map_ptrloc const idxString = WriteStringInMap(S,Size);
 897    if (unlikely(idxString == 0))
 898       return 0;
 899    if (oldMap != Map.Data()) {
 900       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 901       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 902    }
 903    *Last = Item;
 904
 905    // Fill in the structure
 906    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 907    ItemP->NextItem = I - Cache.StringItemP;
 908    ItemP->String = idxString;
 909
 910    Bucket = ItemP;
 911    return ItemP->String;
 912 }
 913                                                                         /*}}}*/
 914 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 915 // ---------------------------------------------------------------------
 916 /* This just verifies that each file in the list of index files exists,
 917    has matching attributes with the cache and the cache does not have
 918    any extra files. */
 919 static bool CheckValidity(const string &CacheFile,
 920                           pkgSourceList &List,
 921                           FileIterator Start,
 922                           FileIterator End,
 923                           MMap **OutMap = 0)
 924 {
 925    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 926    // No file, certainly invalid
 927    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 928    {
 929       if (Debug == true)
 930          std::clog << "CacheFile doesn't exist" << std::endl;
 931       return false;
 932    }
 933
 934    if (List.GetLastModifiedTime() < GetModificationTime(CacheFile))
 935    {
 936       if (Debug == true)
 937          std::clog << "sources.list is newer than the cache" << std::endl;
 938       return false;
 939    }
 940
 941    // Map it
 942    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 943    SPtr<MMap> Map = new MMap(CacheF,0);
 944    pkgCache Cache(Map);
 945    if (_error->PendingError() == true || Map->Size() == 0)
 946    {
 947       if (Debug == true)
 948          std::clog << "Errors are pending or Map is empty()" << std::endl;
 949       _error->Discard();
 950       return false;
 951    }
 952
 953    /* Now we check every index file, see if it is in the cache,
 954       verify the IMS data and check that it is on the disk too.. */
 955    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 956    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 957    for (; Start != End; Start++)
 958    {
 959       if (Debug == true)
 960          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 961       if ((*Start)->HasPackages() == false)
 962       {
 963          if (Debug == true)
 964             std::clog << "Has NO packages" << std::endl;
 965          continue;
 966       }
 967
 968       if ((*Start)->Exists() == false)
 969       {
 970 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 971          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 972                           (*Start)->Describe().c_str());
 973 #endif
 974          if (Debug == true)
 975             std::clog << "file doesn't exist" << std::endl;
 976          continue;
 977       }
 978
 979       // FindInCache is also expected to do an IMS check.
 980       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 981       if (File.end() == true)
 982       {
 983          if (Debug == true)
 984             std::clog << "FindInCache returned end-Pointer" << std::endl;
 985          return false;
 986       }
 987
 988       Visited[File->ID] = true;
 989       if (Debug == true)
 990          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 991    }
 992
 993    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 994       if (Visited[I] == false)
 995       {
 996          if (Debug == true)
 997             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 998          return false;
 999       }
1000
1001    if (_error->PendingError() == true)
1002    {
1003       if (Debug == true)
1004       {
1005          std::clog << "Validity failed because of pending errors:" << std::endl;
1006          _error->DumpErrors();
1007       }
1008       _error->Discard();
1009       return false;
1010    }
1011
1012    if (OutMap != 0)
1013       *OutMap = Map.UnGuard();
1014    return true;
1015 }
1016                                                                         /*}}}*/
1017 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1018 // ---------------------------------------------------------------------
1019 /* Size is kind of an abstract notion that is only used for the progress
1020    meter */
1021 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1022 {
1023    unsigned long TotalSize = 0;
1024    for (; Start != End; Start++)
1025    {
1026       if ((*Start)->HasPackages() == false)
1027          continue;
1028       TotalSize += (*Start)->Size();
1029    }
1030    return TotalSize;
1031 }
1032                                                                         /*}}}*/
1033 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1034 // ---------------------------------------------------------------------
1035 /* */
1036 static bool BuildCache(pkgCacheGenerator &Gen,
1037                        OpProgress *Progress,
1038                        unsigned long &CurrentSize,unsigned long TotalSize,
1039                        FileIterator Start, FileIterator End)
1040 {
1041    FileIterator I;
1042    for (I = Start; I != End; I++)
1043    {
1044       if ((*I)->HasPackages() == false)
1045          continue;
1046
1047       if ((*I)->Exists() == false)
1048          continue;
1049
1050       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1051       {
1052          _error->Warning("Duplicate sources.list entry %s",
1053                          (*I)->Describe().c_str());
1054          continue;
1055       }
1056
1057       unsigned long Size = (*I)->Size();
1058       if (Progress != NULL)
1059          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1060       CurrentSize += Size;
1061
1062       if ((*I)->Merge(Gen,Progress) == false)
1063          return false;
1064    }
1065
1066    if (Gen.HasFileDeps() == true)
1067    {
1068       if (Progress != NULL)
1069          Progress->Done();
1070       TotalSize = ComputeSize(Start, End);
1071       CurrentSize = 0;
1072       for (I = Start; I != End; I++)
1073       {
1074          unsigned long Size = (*I)->Size();
1075          if (Progress != NULL)
1076             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1077          CurrentSize += Size;
1078          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1079             return false;
1080       }
1081    }
1082
1083    return true;
1084 }
1085                                                                         /*}}}*/
1086 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1087 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1088    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1089    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1090    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1091    Flags |= MMap::Moveable;
1092    if (_config->FindB("APT::Cache-Fallback", false) == true)
1093       Flags |= MMap::Fallback;
1094    if (CacheF != NULL)
1095       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1096    else
1097       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1098 }
1099                                                                         /*}}}*/
1100 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1101 // ---------------------------------------------------------------------
1102 /* This makes sure that the status cache (the cache that has all
1103    index files from the sources list and all local ones) is ready
1104    to be mmaped. If OutMap is not zero then a MMap object representing
1105    the cache will be stored there. This is pretty much mandetory if you
1106    are using AllowMem. AllowMem lets the function be run as non-root
1107    where it builds the cache 'fast' into a memory buffer. */
1108 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1109                         MMap **OutMap, bool AllowMem)
1110    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1111 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1112                         MMap **OutMap,bool AllowMem)
1113 {
1114    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1115
1116    vector<pkgIndexFile *> Files;
1117    for (vector<metaIndex *>::const_iterator i = List.begin();
1118         i != List.end();
1119         i++)
1120    {
1121       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1122       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1123            j != Indexes->end();
1124            j++)
1125          Files.push_back (*j);
1126    }
1127
1128    unsigned long const EndOfSource = Files.size();
1129    if (_system->AddStatusFiles(Files) == false)
1130       return false;
1131
1132    // Decide if we can write to the files..
1133    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1134    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1135
1136    // ensure the cache directory exists
1137    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1138    {
1139       string dir = _config->FindDir("Dir::Cache");
1140       size_t const len = dir.size();
1141       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1142          dir = dir.substr(0, len - 5);
1143       if (CacheFile.empty() == false)
1144          CreateDirectory(dir, flNotFile(CacheFile));
1145       if (SrcCacheFile.empty() == false)
1146          CreateDirectory(dir, flNotFile(SrcCacheFile));
1147    }
1148
1149    // Decide if we can write to the cache
1150    bool Writeable = false;
1151    if (CacheFile.empty() == false)
1152       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1153    else
1154       if (SrcCacheFile.empty() == false)
1155          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1156    if (Debug == true)
1157       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1158
1159    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1160       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1161
1162    if (Progress != NULL)
1163       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1164
1165    // Cache is OK, Fin.
1166    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1167    {
1168       if (Progress != NULL)
1169          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1170       if (Debug == true)
1171          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1172       return true;
1173    }
1174    else if (Debug == true)
1175          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1176
1177    /* At this point we know we need to reconstruct the package cache,
1178       begin. */
1179    SPtr<FileFd> CacheF;
1180    SPtr<DynamicMMap> Map;
1181    if (Writeable == true && CacheFile.empty() == false)
1182    {
1183       _error->PushToStack();
1184       unlink(CacheFile.c_str());
1185       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1186       fchmod(CacheF->Fd(),0644);
1187       Map = CreateDynamicMMap(CacheF, MMap::Public);
1188       if (_error->PendingError() == true)
1189       {
1190          delete CacheF.UnGuard();
1191          delete Map.UnGuard();
1192          if (Debug == true)
1193             std::clog << "Open filebased MMap FAILED" << std::endl;
1194          Writeable = false;
1195          if (AllowMem == false)
1196          {
1197             _error->MergeWithStack();
1198             return false;
1199          }
1200          _error->RevertToStack();
1201       }
1202       else if (Debug == true)
1203       {
1204          _error->MergeWithStack();
1205          std::clog << "Open filebased MMap" << std::endl;
1206       }
1207    }
1208    if (Writeable == false || CacheFile.empty() == true)
1209    {
1210       // Just build it in memory..
1211       Map = CreateDynamicMMap(NULL);
1212       if (Debug == true)
1213          std::clog << "Open memory Map (not filebased)" << std::endl;
1214    }
1215
1216    // Lets try the source cache.
1217    unsigned long CurrentSize = 0;
1218    unsigned long TotalSize = 0;
1219    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1220                      Files.begin()+EndOfSource) == true)
1221    {
1222       if (Debug == true)
1223          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1224       // Preload the map with the source cache
1225       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1226       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1227       if ((alloc == 0 && _error->PendingError())
1228                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1229                                 SCacheF.Size()) == false)
1230          return false;
1231
1232       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1233
1234       // Build the status cache
1235       pkgCacheGenerator Gen(Map.Get(),Progress);
1236       if (_error->PendingError() == true)
1237          return false;
1238       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1239                      Files.begin()+EndOfSource,Files.end()) == false)
1240          return false;
1241
1242       // FIXME: move me to a better place
1243       Gen.FinishCache(Progress);
1244    }
1245    else
1246    {
1247       if (Debug == true)
1248          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1249       TotalSize = ComputeSize(Files.begin(),Files.end());
1250
1251       // Build the source cache
1252       pkgCacheGenerator Gen(Map.Get(),Progress);
1253       if (_error->PendingError() == true)
1254          return false;
1255       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1256                      Files.begin(),Files.begin()+EndOfSource) == false)
1257          return false;
1258
1259       // Write it back
1260       if (Writeable == true && SrcCacheFile.empty() == false)
1261       {
1262          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1263          if (_error->PendingError() == true)
1264             return false;
1265
1266          fchmod(SCacheF.Fd(),0644);
1267
1268          // Write out the main data
1269          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1270             return _error->Error(_("IO Error saving source cache"));
1271          SCacheF.Sync();
1272
1273          // Write out the proper header
1274          Gen.GetCache().HeaderP->Dirty = false;
1275          if (SCacheF.Seek(0) == false ||
1276              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1277             return _error->Error(_("IO Error saving source cache"));
1278          Gen.GetCache().HeaderP->Dirty = true;
1279          SCacheF.Sync();
1280       }
1281
1282       // Build the status cache
1283       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1284                      Files.begin()+EndOfSource,Files.end()) == false)
1285          return false;
1286
1287       // FIXME: move me to a better place
1288       Gen.FinishCache(Progress);
1289    }
1290    if (Debug == true)
1291       std::clog << "Caches are ready for shipping" << std::endl;
1292
1293    if (_error->PendingError() == true)
1294       return false;
1295    if (OutMap != 0)
1296    {
1297       if (CacheF != 0)
1298       {
1299          delete Map.UnGuard();
1300          *OutMap = new MMap(*CacheF,0);
1301       }
1302       else
1303       {
1304          *OutMap = Map.UnGuard();
1305       }
1306    }
1307
1308    return true;
1309 }
1310                                                                         /*}}}*/
1311 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1312 // ---------------------------------------------------------------------
1313 /* */
1314 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1315    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1316 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1317 {
1318    vector<pkgIndexFile *> Files;
1319    unsigned long EndOfSource = Files.size();
1320    if (_system->AddStatusFiles(Files) == false)
1321       return false;
1322
1323    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1324    unsigned long CurrentSize = 0;
1325    unsigned long TotalSize = 0;
1326
1327    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1328
1329    // Build the status cache
1330    if (Progress != NULL)
1331       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1332    pkgCacheGenerator Gen(Map.Get(),Progress);
1333    if (_error->PendingError() == true)
1334       return false;
1335    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1336                   Files.begin()+EndOfSource,Files.end()) == false)
1337       return false;
1338
1339    // FIXME: move me to a better place
1340    Gen.FinishCache(Progress);
1341
1342    if (_error->PendingError() == true)
1343       return false;
1344    *OutMap = Map.UnGuard();
1345
1346    return true;
1347 }
1348                                                                         /*}}}*/