apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26
  27 #include <apt-pkg/tagfile.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Map.Sync(0,sizeof(pkgCache::Header));
 102 }
 103                                                                         /*}}}*/
 104 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 105    if (oldMap == newMap)
 106       return;
 107
 108    Cache.ReMap(false);
 109
 110    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 111
 112    for (size_t i = 0; i < _count(UniqHash); ++i)
 113       if (UniqHash[i] != 0)
 114          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 115
 116    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 117         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 118       (*i)->ReMap(oldMap, newMap);
 119    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 120         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 121       (*i)->ReMap(oldMap, newMap);
 122    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 123         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 124       (*i)->ReMap(oldMap, newMap);
 125    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 126         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 127       (*i)->ReMap(oldMap, newMap);
 128    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 129         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 130       (*i)->ReMap(oldMap, newMap);
 131    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 132         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 133       (*i)->ReMap(oldMap, newMap);
 134    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 135         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 136       (*i)->ReMap(oldMap, newMap);
 137 }                                                                       /*}}}*/
 138 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 139 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 140                                         const unsigned long &Len) {
 141    void const * const oldMap = Map.Data();
 142    map_ptrloc const index = Map.WriteString(String, Len);
 143    if (index != 0)
 144       ReMap(oldMap, Map.Data());
 145    return index;
 146 }
 147                                                                         /*}}}*/
 148 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 149 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 150    void const * const oldMap = Map.Data();
 151    map_ptrloc const index = Map.WriteString(String);
 152    if (index != 0)
 153       ReMap(oldMap, Map.Data());
 154    return index;
 155 }
 156                                                                         /*}}}*/
 157 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 158    void const * const oldMap = Map.Data();
 159    map_ptrloc const index = Map.Allocate(size);
 160    if (index != 0)
 161       ReMap(oldMap, Map.Data());
 162    return index;
 163 }
 164                                                                         /*}}}*/
 165 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* This provides the generation of the entries in the cache. Each loop
 168    goes through a single package record from the underlying parse engine. */
 169 bool pkgCacheGenerator::MergeList(ListParser &List,
 170                                   pkgCache::VerIterator *OutVer)
 171 {
 172    List.Owner = this;
 173
 174    unsigned int Counter = 0;
 175    while (List.Step() == true)
 176    {
 177       string const PackageName = List.Package();
 178       if (PackageName.empty() == true)
 179          return false;
 180
 181       /* Treat Arch all packages as the same as the native arch. */
 182       string Arch;
 183       if (List.ArchitectureAll() == true)
 184          Arch = _config->Find("APT::Architecture");
 185       else
 186          Arch = List.Architecture();
 187
 188       // Get a pointer to the package structure
 189       pkgCache::PkgIterator Pkg;
 190       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 191       if (NewPackage(Pkg, PackageName, Arch) == false)
 192          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 193       Counter++;
 194       if (Counter % 100 == 0 && Progress != 0)
 195          Progress->Progress(List.Offset());
 196
 197       /* Get a pointer to the version structure. We know the list is sorted
 198          so we use that fact in the search. Insertion of new versions is
 199          done with correct sorting */
 200       string Version = List.Version();
 201       if (Version.empty() == true)
 202       {
 203          // we first process the package, then the descriptions
 204          // (this has the bonus that we get MMap error when we run out
 205          //  of MMap space)
 206          pkgCache::VerIterator Ver(Cache);
 207          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 208          if (List.UsePackage(Pkg, Ver) == false)
 209             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 210                                  PackageName.c_str());
 211
 212          // Find the right version to write the description
 213          MD5SumValue CurMd5 = List.Description_md5();
 214          Ver = Pkg.VersionList();
 215
 216          for (; Ver.end() == false; ++Ver)
 217          {
 218             pkgCache::DescIterator Desc = Ver.DescriptionList();
 219             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 220             map_ptrloc *LastDesc = &Ver->DescriptionList;
 221             bool duplicate=false;
 222
 223             // don't add a new description if we have one for the given
 224             // md5 && language
 225             for ( ; Desc.end() == false; Desc++)
 226                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 227                    Desc.LanguageCode() == List.DescriptionLanguage())
 228                   duplicate=true;
 229             if(duplicate)
 230                continue;
 231
 232             for (Desc = Ver.DescriptionList();
 233                  Desc.end() == false;
 234                  LastDesc = &Desc->NextDesc, Desc++)
 235             {
 236                if (MD5SumValue(Desc.md5()) == CurMd5)
 237                {
 238                   // Add new description
 239                   void const * const oldMap = Map.Data();
 240                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 241                   if (oldMap != Map.Data())
 242                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 243                   *LastDesc = descindex;
 244                   Desc->ParentPkg = Pkg.Index();
 245
 246                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 247                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 248                   break;
 249                }
 250             }
 251          }
 252
 253          continue;
 254       }
 255
 256       pkgCache::VerIterator Ver = Pkg.VersionList();
 257       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 258       map_ptrloc *LastVer = &Pkg->VersionList;
 259       void const * oldMap = Map.Data();
 260       int Res = 1;
 261       unsigned long const Hash = List.VersionHash();
 262       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 263       {
 264          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 265          // Version is higher as current version - insert here
 266          if (Res > 0)
 267             break;
 268          // Versionstrings are equal - is hash also equal?
 269          if (Res == 0 && Ver->Hash == Hash)
 270             break;
 271          // proceed with the next till we have either the right
 272          // or we found another version (which will be lower)
 273       }
 274
 275       /* We already have a version for this item, record that we saw it */
 276       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 277       {
 278          if (List.UsePackage(Pkg,Ver) == false)
 279             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 280                                  PackageName.c_str());
 281
 282          if (NewFileVer(Ver,List) == false)
 283             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 284                                  PackageName.c_str());
 285
 286          // Read only a single record and return
 287          if (OutVer != 0)
 288          {
 289             *OutVer = Ver;
 290             FoundFileDeps |= List.HasFileDeps();
 291             return true;
 292          }
 293
 294          continue;
 295       }
 296
 297       // Add a new version
 298       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 299       if (verindex == 0 && _error->PendingError())
 300          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 301                               PackageName.c_str(), 1);
 302
 303       if (oldMap != Map.Data())
 304          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 305       *LastVer = verindex;
 306       Ver->ParentPkg = Pkg.Index();
 307       Ver->Hash = Hash;
 308
 309       if (List.NewVersion(Ver) == false)
 310          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 311                               PackageName.c_str(), 2);
 312
 313       if (List.UsePackage(Pkg,Ver) == false)
 314          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 315                               PackageName.c_str());
 316
 317       if (NewFileVer(Ver,List) == false)
 318          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 319                               PackageName.c_str(), 3);
 320
 321       // Read only a single record and return
 322       if (OutVer != 0)
 323       {
 324          *OutVer = Ver;
 325          FoundFileDeps |= List.HasFileDeps();
 326          return true;
 327       }
 328
 329       /* Record the Description data. Description data always exist in
 330          Packages and Translation-* files. */
 331       pkgCache::DescIterator Desc = Ver.DescriptionList();
 332       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 333       map_ptrloc *LastDesc = &Ver->DescriptionList;
 334
 335       // Skip to the end of description set
 336       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 337
 338       // Add new description
 339       oldMap = Map.Data();
 340       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 341       if (oldMap != Map.Data())
 342          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 343       *LastDesc = descindex;
 344       Desc->ParentPkg = Pkg.Index();
 345
 346       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 347          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 348    }
 349
 350    FoundFileDeps |= List.HasFileDeps();
 351
 352    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 353       return _error->Error(_("Wow, you exceeded the number of package "
 354                              "names this APT is capable of."));
 355    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 356       return _error->Error(_("Wow, you exceeded the number of versions "
 357                              "this APT is capable of."));
 358    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 359       return _error->Error(_("Wow, you exceeded the number of descriptions "
 360                              "this APT is capable of."));
 361    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 362       return _error->Error(_("Wow, you exceeded the number of dependencies "
 363                              "this APT is capable of."));
 364    return true;
 365 }
 366                                                                         /*}}}*/
 367 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* If we found any file depends while parsing the main list we need to
 370    resolve them. Since it is undesired to load the entire list of files
 371    into the cache as virtual packages we do a two stage effort. MergeList
 372    identifies the file depends and this creates Provdies for them by
 373    re-parsing all the indexs. */
 374 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 375 {
 376    List.Owner = this;
 377
 378    unsigned int Counter = 0;
 379    while (List.Step() == true)
 380    {
 381       string PackageName = List.Package();
 382       if (PackageName.empty() == true)
 383          return false;
 384       string Version = List.Version();
 385       if (Version.empty() == true)
 386          continue;
 387
 388       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 389       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 390       if (Pkg.end() == true)
 391          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 392                                 PackageName.c_str());
 393       Counter++;
 394       if (Counter % 100 == 0 && Progress != 0)
 395          Progress->Progress(List.Offset());
 396
 397       unsigned long Hash = List.VersionHash();
 398       pkgCache::VerIterator Ver = Pkg.VersionList();
 399       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 400       for (; Ver.end() == false; Ver++)
 401       {
 402          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 403          {
 404             if (List.CollectFileProvides(Cache,Ver) == false)
 405                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 406             break;
 407          }
 408       }
 409
 410       if (Ver.end() == true)
 411          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 412    }
 413
 414    return true;
 415 }
 416                                                                         /*}}}*/
 417 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 418 // ---------------------------------------------------------------------
 419 /* This creates a new group structure and adds it to the hash table */
 420 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 421 {
 422    Grp = Cache.FindGrp(Name);
 423    if (Grp.end() == false)
 424       return true;
 425
 426    // Get a structure
 427    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 428    if (unlikely(Group == 0))
 429       return false;
 430
 431    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 432    map_ptrloc const idxName = WriteStringInMap(Name);
 433    if (unlikely(idxName == 0))
 434       return false;
 435    Grp->Name = idxName;
 436
 437    // Insert it into the hash table
 438    unsigned long const Hash = Cache.Hash(Name);
 439    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 440    Cache.HeaderP->GrpHashTable[Hash] = Group;
 441
 442    Grp->ID = Cache.HeaderP->GroupCount++;
 443    return true;
 444 }
 445                                                                         /*}}}*/
 446 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* This creates a new package structure and adds it to the hash table */
 449 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 450                                         const string &Arch) {
 451    pkgCache::GrpIterator Grp;
 452    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 453    if (unlikely(NewGroup(Grp, Name) == false))
 454       return false;
 455
 456    Pkg = Grp.FindPkg(Arch);
 457       if (Pkg.end() == false)
 458          return true;
 459
 460    // Get a structure
 461    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 462    if (unlikely(Package == 0))
 463       return false;
 464    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 465
 466    // Insert the package into our package list
 467    if (Grp->FirstPackage == 0) // the group is new
 468    {
 469       // Insert it into the hash table
 470       unsigned long const Hash = Cache.Hash(Name);
 471       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 472       Cache.HeaderP->PkgHashTable[Hash] = Package;
 473       Grp->FirstPackage = Package;
 474    }
 475    else // Group the Packages together
 476    {
 477       // this package is the new last package
 478       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 479       Pkg->NextPackage = LastPkg->NextPackage;
 480       LastPkg->NextPackage = Package;
 481    }
 482    Grp->LastPackage = Package;
 483
 484    // Set the name, arch and the ID
 485    Pkg->Name = Grp->Name;
 486    Pkg->Group = Grp.Index();
 487    map_ptrloc const idxArch = WriteUniqString(Arch.c_str());
 488    if (unlikely(idxArch == 0))
 489       return false;
 490    Pkg->Arch = idxArch;
 491    Pkg->ID = Cache.HeaderP->PackageCount++;
 492
 493    return true;
 494 }
 495                                                                         /*}}}*/
 496 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* */
 499 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 500                                    ListParser &List)
 501 {
 502    if (CurrentFile == 0)
 503       return true;
 504
 505    // Get a structure
 506    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 507    if (VerFile == 0)
 508       return 0;
 509
 510    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 511    VF->File = CurrentFile - Cache.PkgFileP;
 512
 513    // Link it to the end of the list
 514    map_ptrloc *Last = &Ver->FileList;
 515    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 516       Last = &V->NextFile;
 517    VF->NextFile = *Last;
 518    *Last = VF.Index();
 519
 520    VF->Offset = List.Offset();
 521    VF->Size = List.Size();
 522    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 523       Cache.HeaderP->MaxVerFileSize = VF->Size;
 524    Cache.HeaderP->VerFileCount++;
 525
 526    return true;
 527 }
 528                                                                         /*}}}*/
 529 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* This puts a version structure in the linked list */
 532 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 533                                             const string &VerStr,
 534                                             unsigned long Next)
 535 {
 536    // Get a structure
 537    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 538    if (Version == 0)
 539       return 0;
 540
 541    // Fill it in
 542    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 543    Ver->NextVer = Next;
 544    Ver->ID = Cache.HeaderP->VersionCount++;
 545    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 546    if (unlikely(idxVerStr == 0))
 547       return 0;
 548    Ver->VerStr = idxVerStr;
 549
 550    return Version;
 551 }
 552                                                                         /*}}}*/
 553 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 554 // ---------------------------------------------------------------------
 555 /* */
 556 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 557                                    ListParser &List)
 558 {
 559    if (CurrentFile == 0)
 560       return true;
 561
 562    // Get a structure
 563    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 564    if (DescFile == 0)
 565       return false;
 566
 567    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 568    DF->File = CurrentFile - Cache.PkgFileP;
 569
 570    // Link it to the end of the list
 571    map_ptrloc *Last = &Desc->FileList;
 572    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 573       Last = &D->NextFile;
 574
 575    DF->NextFile = *Last;
 576    *Last = DF.Index();
 577
 578    DF->Offset = List.Offset();
 579    DF->Size = List.Size();
 580    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 581       Cache.HeaderP->MaxDescFileSize = DF->Size;
 582    Cache.HeaderP->DescFileCount++;
 583
 584    return true;
 585 }
 586                                                                         /*}}}*/
 587 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 588 // ---------------------------------------------------------------------
 589 /* This puts a description structure in the linked list */
 590 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 591                                             const string &Lang,
 592                                             const MD5SumValue &md5sum,
 593                                             map_ptrloc Next)
 594 {
 595    // Get a structure
 596    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 597    if (Description == 0)
 598       return 0;
 599
 600    // Fill it in
 601    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 602    Desc->NextDesc = Next;
 603    Desc->ID = Cache.HeaderP->DescriptionCount++;
 604    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 605    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 606    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 607       return 0;
 608    Desc->language_code = idxlanguage_code;
 609    Desc->md5sum = idxmd5sum;
 610
 611    return Description;
 612 }
 613                                                                         /*}}}*/
 614 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 615 // ---------------------------------------------------------------------
 616 /* This prepares the Cache for delivery */
 617 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 618 {
 619    // FIXME: add progress reporting for this operation
 620    // Do we have different architectures in your groups ?
 621    vector<string> archs = APT::Configuration::getArchitectures();
 622    if (archs.size() > 1)
 623    {
 624       // Create Conflicts in between the group
 625       pkgCache::GrpIterator G = GetCache().GrpBegin();
 626       Dynamic<pkgCache::GrpIterator> DynG(G);
 627       for (; G.end() != true; G++)
 628       {
 629          string const PkgName = G.Name();
 630          pkgCache::PkgIterator P = G.PackageList();
 631          Dynamic<pkgCache::PkgIterator> DynP(P);
 632          for (; P.end() != true; P = G.NextPkg(P))
 633          {
 634             pkgCache::PkgIterator allPkg;
 635             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 636             pkgCache::VerIterator V = P.VersionList();
 637             Dynamic<pkgCache::VerIterator> DynV(V);
 638             for (; V.end() != true; V++)
 639             {
 640                char const * const Arch = P.Arch();
 641                map_ptrloc *OldDepLast = NULL;
 642                /* MultiArch handling introduces a lot of implicit Dependencies:
 643                 - MultiArch: same → Co-Installable if they have the same version
 644                 - Architecture: all → Need to be Co-Installable for internal reasons
 645                 - All others conflict with all other group members */
 646                bool const coInstall = (V->MultiArch == pkgCache::Version::Same);
 647                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 648                {
 649                   if (*A == Arch)
 650                      continue;
 651                   /* We allow only one installed arch at the time
 652                      per group, therefore each group member conflicts
 653                      with all other group members */
 654                   pkgCache::PkgIterator D = G.FindPkg(*A);
 655                   Dynamic<pkgCache::PkgIterator> DynD(D);
 656                   if (D.end() == true)
 657                      continue;
 658                   if (coInstall == true)
 659                   {
 660                      // Replaces: ${self}:other ( << ${binary:Version})
 661                      NewDepends(D, V, V.VerStr(),
 662                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 663                                 OldDepLast);
 664                      // Breaks: ${self}:other (!= ${binary:Version})
 665                      NewDepends(D, V, V.VerStr(),
 666                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 667                                 OldDepLast);
 668                   } else {
 669                         // Conflicts: ${self}:other
 670                         NewDepends(D, V, "",
 671                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 672                                    OldDepLast);
 673                   }
 674                }
 675             }
 676          }
 677       }
 678    }
 679    return true;
 680 }
 681                                                                         /*}}}*/
 682 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 683 // ---------------------------------------------------------------------
 684 /* This creates a dependency element in the tree. It is linked to the
 685    version and to the package that it is pointing to. */
 686 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 687                                    pkgCache::VerIterator &Ver,
 688                                    string const &Version,
 689                                    unsigned int const &Op,
 690                                    unsigned int const &Type,
 691                                    map_ptrloc *OldDepLast)
 692 {
 693    void const * const oldMap = Map.Data();
 694    // Get a structure
 695    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 696    if (unlikely(Dependency == 0))
 697       return false;
 698
 699    // Fill it in
 700    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 701    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 702    Dep->ParentVer = Ver.Index();
 703    Dep->Type = Type;
 704    Dep->CompareOp = Op;
 705    Dep->ID = Cache.HeaderP->DependsCount++;
 706
 707    // Probe the reverse dependency list for a version string that matches
 708    if (Version.empty() == false)
 709    {
 710 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 711          if (I->Version != 0 && I.TargetVer() == Version)
 712             Dep->Version = I->Version;*/
 713       if (Dep->Version == 0) {
 714          map_ptrloc const index = WriteStringInMap(Version);
 715          if (unlikely(index == 0))
 716             return false;
 717          Dep->Version = index;
 718       }
 719    }
 720
 721    // Link it to the package
 722    Dep->Package = Pkg.Index();
 723    Dep->NextRevDepends = Pkg->RevDepends;
 724    Pkg->RevDepends = Dep.Index();
 725
 726    // Do we know where to link the Dependency to?
 727    if (OldDepLast == NULL)
 728    {
 729       OldDepLast = &Ver->DependsList;
 730       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 731          OldDepLast = &D->NextDepends;
 732    } else if (oldMap != Map.Data())
 733       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 734
 735    Dep->NextDepends = *OldDepLast;
 736    *OldDepLast = Dep.Index();
 737    OldDepLast = &Dep->NextDepends;
 738
 739    return true;
 740 }
 741                                                                         /*}}}*/
 742 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 743 // ---------------------------------------------------------------------
 744 /* This creates a Group and the Package to link this dependency to if
 745    needed and handles also the caching of the old endpoint */
 746 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 747                                                const string &PackageName,
 748                                                const string &Arch,
 749                                                const string &Version,
 750                                                unsigned int Op,
 751                                                unsigned int Type)
 752 {
 753    pkgCache::GrpIterator Grp;
 754    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 755    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 756       return false;
 757
 758    // Locate the target package
 759    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 760    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 761    if (Pkg.end() == true) {
 762       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 763          return false;
 764    }
 765
 766    // Is it a file dependency?
 767    if (unlikely(PackageName[0] == '/'))
 768       FoundFileDeps = true;
 769
 770    /* Caching the old end point speeds up generation substantially */
 771    if (OldDepVer != Ver) {
 772       OldDepLast = NULL;
 773       OldDepVer = Ver;
 774    }
 775
 776    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 777 }
 778                                                                         /*}}}*/
 779 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 780 // ---------------------------------------------------------------------
 781 /* */
 782 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 783                                                 const string &PkgName,
 784                                                 const string &PkgArch,
 785                                                 const string &Version)
 786 {
 787    pkgCache &Cache = Owner->Cache;
 788
 789    // We do not add self referencing provides
 790    if (Ver.ParentPkg().Name() == PkgName && PkgArch == Ver.Arch())
 791       return true;
 792
 793    // Get a structure
 794    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 795    if (unlikely(Provides == 0))
 796       return false;
 797    Cache.HeaderP->ProvidesCount++;
 798
 799    // Fill it in
 800    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 801    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 802    Prv->Version = Ver.Index();
 803    Prv->NextPkgProv = Ver->ProvidesList;
 804    Ver->ProvidesList = Prv.Index();
 805    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 806       return false;
 807
 808    // Locate the target package
 809    pkgCache::PkgIterator Pkg;
 810    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 811    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 812       return false;
 813
 814    // Link it to the package
 815    Prv->ParentPkg = Pkg.Index();
 816    Prv->NextProvides = Pkg->ProvidesList;
 817    Pkg->ProvidesList = Prv.Index();
 818
 819    return true;
 820 }
 821                                                                         /*}}}*/
 822 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 823 // ---------------------------------------------------------------------
 824 /* This is used to select which file is to be associated with all newly
 825    added versions. The caller is responsible for setting the IMS fields. */
 826 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 827                                    const pkgIndexFile &Index,
 828                                    unsigned long Flags)
 829 {
 830    // Get some space for the structure
 831    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 832    if (unlikely(idxFile == 0))
 833       return false;
 834    CurrentFile = Cache.PkgFileP + idxFile;
 835
 836    // Fill it in
 837    map_ptrloc const idxFileName = WriteStringInMap(File);
 838    map_ptrloc const idxSite = WriteUniqString(Site);
 839    if (unlikely(idxFileName == 0 || idxSite == 0))
 840       return false;
 841    CurrentFile->FileName = idxFileName;
 842    CurrentFile->Site = idxSite;
 843    CurrentFile->NextFile = Cache.HeaderP->FileList;
 844    CurrentFile->Flags = Flags;
 845    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 846    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 847    if (unlikely(idxIndexType == 0))
 848       return false;
 849    CurrentFile->IndexType = idxIndexType;
 850    PkgFileName = File;
 851    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 852    Cache.HeaderP->PackageFileCount++;
 853
 854    if (Progress != 0)
 855       Progress->SubProgress(Index.Size());
 856    return true;
 857 }
 858                                                                         /*}}}*/
 859 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 860 // ---------------------------------------------------------------------
 861 /* This is used to create handles to strings. Given the same text it
 862    always returns the same number */
 863 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 864                                                  unsigned int Size)
 865 {
 866    /* We use a very small transient hash table here, this speeds up generation
 867       by a fair amount on slower machines */
 868    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 869    if (Bucket != 0 &&
 870        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 871       return Bucket->String;
 872
 873    // Search for an insertion point
 874    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 875    int Res = 1;
 876    map_ptrloc *Last = &Cache.HeaderP->StringList;
 877    for (; I != Cache.StringItemP; Last = &I->NextItem,
 878         I = Cache.StringItemP + I->NextItem)
 879    {
 880       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 881       if (Res >= 0)
 882          break;
 883    }
 884
 885    // Match
 886    if (Res == 0)
 887    {
 888       Bucket = I;
 889       return I->String;
 890    }
 891
 892    // Get a structure
 893    void const * const oldMap = Map.Data();
 894    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 895    if (Item == 0)
 896       return 0;
 897
 898    map_ptrloc const idxString = WriteStringInMap(S,Size);
 899    if (unlikely(idxString == 0))
 900       return 0;
 901    if (oldMap != Map.Data()) {
 902       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 903       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 904    }
 905    *Last = Item;
 906
 907    // Fill in the structure
 908    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 909    ItemP->NextItem = I - Cache.StringItemP;
 910    ItemP->String = idxString;
 911
 912    Bucket = ItemP;
 913    return ItemP->String;
 914 }
 915                                                                         /*}}}*/
 916 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 917 // ---------------------------------------------------------------------
 918 /* This just verifies that each file in the list of index files exists,
 919    has matching attributes with the cache and the cache does not have
 920    any extra files. */
 921 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 922                           FileIterator End,MMap **OutMap = 0)
 923 {
 924    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 925    // No file, certainly invalid
 926    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 927    {
 928       if (Debug == true)
 929          std::clog << "CacheFile doesn't exist" << std::endl;
 930       return false;
 931    }
 932
 933    // Map it
 934    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 935    SPtr<MMap> Map = new MMap(CacheF,0);
 936    pkgCache Cache(Map);
 937    if (_error->PendingError() == true || Map->Size() == 0)
 938    {
 939       if (Debug == true)
 940          std::clog << "Errors are pending or Map is empty()" << std::endl;
 941       _error->Discard();
 942       return false;
 943    }
 944
 945    /* Now we check every index file, see if it is in the cache,
 946       verify the IMS data and check that it is on the disk too.. */
 947    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 948    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 949    for (; Start != End; Start++)
 950    {
 951       if (Debug == true)
 952          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 953       if ((*Start)->HasPackages() == false)
 954       {
 955          if (Debug == true)
 956             std::clog << "Has NO packages" << std::endl;
 957          continue;
 958       }
 959
 960       if ((*Start)->Exists() == false)
 961       {
 962 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 963          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 964                           (*Start)->Describe().c_str());
 965 #endif
 966          if (Debug == true)
 967             std::clog << "file doesn't exist" << std::endl;
 968          continue;
 969       }
 970
 971       // FindInCache is also expected to do an IMS check.
 972       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 973       if (File.end() == true)
 974       {
 975          if (Debug == true)
 976             std::clog << "FindInCache returned end-Pointer" << std::endl;
 977          return false;
 978       }
 979
 980       Visited[File->ID] = true;
 981       if (Debug == true)
 982          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 983    }
 984
 985    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 986       if (Visited[I] == false)
 987       {
 988          if (Debug == true)
 989             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 990          return false;
 991       }
 992
 993    if (_error->PendingError() == true)
 994    {
 995       if (Debug == true)
 996       {
 997          std::clog << "Validity failed because of pending errors:" << std::endl;
 998          _error->DumpErrors();
 999       }
1000       _error->Discard();
1001       return false;
1002    }
1003
1004    if (OutMap != 0)
1005       *OutMap = Map.UnGuard();
1006    return true;
1007 }
1008                                                                         /*}}}*/
1009 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1010 // ---------------------------------------------------------------------
1011 /* Size is kind of an abstract notion that is only used for the progress
1012    meter */
1013 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1014 {
1015    unsigned long TotalSize = 0;
1016    for (; Start != End; Start++)
1017    {
1018       if ((*Start)->HasPackages() == false)
1019          continue;
1020       TotalSize += (*Start)->Size();
1021    }
1022    return TotalSize;
1023 }
1024                                                                         /*}}}*/
1025 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1026 // ---------------------------------------------------------------------
1027 /* */
1028 static bool BuildCache(pkgCacheGenerator &Gen,
1029                        OpProgress *Progress,
1030                        unsigned long &CurrentSize,unsigned long TotalSize,
1031                        FileIterator Start, FileIterator End)
1032 {
1033    FileIterator I;
1034    for (I = Start; I != End; I++)
1035    {
1036       if ((*I)->HasPackages() == false)
1037          continue;
1038
1039       if ((*I)->Exists() == false)
1040          continue;
1041
1042       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1043       {
1044          _error->Warning("Duplicate sources.list entry %s",
1045                          (*I)->Describe().c_str());
1046          continue;
1047       }
1048
1049       unsigned long Size = (*I)->Size();
1050       if (Progress != NULL)
1051          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1052       CurrentSize += Size;
1053
1054       if ((*I)->Merge(Gen,Progress) == false)
1055          return false;
1056    }
1057
1058    if (Gen.HasFileDeps() == true)
1059    {
1060       if (Progress != NULL)
1061          Progress->Done();
1062       TotalSize = ComputeSize(Start, End);
1063       CurrentSize = 0;
1064       for (I = Start; I != End; I++)
1065       {
1066          unsigned long Size = (*I)->Size();
1067          if (Progress != NULL)
1068             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1069          CurrentSize += Size;
1070          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1071             return false;
1072       }
1073    }
1074
1075    return true;
1076 }
1077                                                                         /*}}}*/
1078 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1079 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1080    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1081    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1082    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1083    Flags |= MMap::Moveable;
1084    if (_config->FindB("APT::Cache-Fallback", false) == true)
1085       Flags |= MMap::Fallback;
1086    if (CacheF != NULL)
1087       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1088    else
1089       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1090 }
1091                                                                         /*}}}*/
1092 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1093 // ---------------------------------------------------------------------
1094 /* This makes sure that the status cache (the cache that has all
1095    index files from the sources list and all local ones) is ready
1096    to be mmaped. If OutMap is not zero then a MMap object representing
1097    the cache will be stored there. This is pretty much mandetory if you
1098    are using AllowMem. AllowMem lets the function be run as non-root
1099    where it builds the cache 'fast' into a memory buffer. */
1100 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1101                         MMap **OutMap, bool AllowMem)
1102    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1103 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1104                         MMap **OutMap,bool AllowMem)
1105 {
1106    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1107
1108    vector<pkgIndexFile *> Files;
1109    for (vector<metaIndex *>::const_iterator i = List.begin();
1110         i != List.end();
1111         i++)
1112    {
1113       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1114       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1115            j != Indexes->end();
1116            j++)
1117          Files.push_back (*j);
1118    }
1119
1120    unsigned long const EndOfSource = Files.size();
1121    if (_system->AddStatusFiles(Files) == false)
1122       return false;
1123
1124    // Decide if we can write to the files..
1125    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1126    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1127
1128    // ensure the cache directory exists
1129    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1130    {
1131       string dir = _config->FindDir("Dir::Cache");
1132       size_t const len = dir.size();
1133       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1134          dir = dir.substr(0, len - 5);
1135       if (CacheFile.empty() == false)
1136          CreateDirectory(dir, flNotFile(CacheFile));
1137       if (SrcCacheFile.empty() == false)
1138          CreateDirectory(dir, flNotFile(SrcCacheFile));
1139    }
1140
1141    // Decide if we can write to the cache
1142    bool Writeable = false;
1143    if (CacheFile.empty() == false)
1144       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1145    else
1146       if (SrcCacheFile.empty() == false)
1147          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1148    if (Debug == true)
1149       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1150
1151    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1152       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1153
1154    if (Progress != NULL)
1155       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1156
1157    // Cache is OK, Fin.
1158    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
1159    {
1160       if (Progress != NULL)
1161          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1162       if (Debug == true)
1163          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1164       return true;
1165    }
1166    else if (Debug == true)
1167          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1168
1169    /* At this point we know we need to reconstruct the package cache,
1170       begin. */
1171    SPtr<FileFd> CacheF;
1172    SPtr<DynamicMMap> Map;
1173    if (Writeable == true && CacheFile.empty() == false)
1174    {
1175       unlink(CacheFile.c_str());
1176       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1177       fchmod(CacheF->Fd(),0644);
1178       Map = CreateDynamicMMap(CacheF, MMap::Public);
1179       if (_error->PendingError() == true)
1180          return false;
1181       if (Debug == true)
1182          std::clog << "Open filebased MMap" << std::endl;
1183    }
1184    else
1185    {
1186       // Just build it in memory..
1187       Map = CreateDynamicMMap(NULL);
1188       if (Debug == true)
1189          std::clog << "Open memory Map (not filebased)" << std::endl;
1190    }
1191
1192    // Lets try the source cache.
1193    unsigned long CurrentSize = 0;
1194    unsigned long TotalSize = 0;
1195    if (CheckValidity(SrcCacheFile,Files.begin(),
1196                      Files.begin()+EndOfSource) == true)
1197    {
1198       if (Debug == true)
1199          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1200       // Preload the map with the source cache
1201       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1202       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1203       if ((alloc == 0 && _error->PendingError())
1204                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1205                                 SCacheF.Size()) == false)
1206          return false;
1207
1208       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1209
1210       // Build the status cache
1211       pkgCacheGenerator Gen(Map.Get(),Progress);
1212       if (_error->PendingError() == true)
1213          return false;
1214       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1215                      Files.begin()+EndOfSource,Files.end()) == false)
1216          return false;
1217
1218       // FIXME: move me to a better place
1219       Gen.FinishCache(Progress);
1220    }
1221    else
1222    {
1223       if (Debug == true)
1224          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1225       TotalSize = ComputeSize(Files.begin(),Files.end());
1226
1227       // Build the source cache
1228       pkgCacheGenerator Gen(Map.Get(),Progress);
1229       if (_error->PendingError() == true)
1230          return false;
1231       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1232                      Files.begin(),Files.begin()+EndOfSource) == false)
1233          return false;
1234
1235       // Write it back
1236       if (Writeable == true && SrcCacheFile.empty() == false)
1237       {
1238          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1239          if (_error->PendingError() == true)
1240             return false;
1241
1242          fchmod(SCacheF.Fd(),0644);
1243
1244          // Write out the main data
1245          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1246             return _error->Error(_("IO Error saving source cache"));
1247          SCacheF.Sync();
1248
1249          // Write out the proper header
1250          Gen.GetCache().HeaderP->Dirty = false;
1251          if (SCacheF.Seek(0) == false ||
1252              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1253             return _error->Error(_("IO Error saving source cache"));
1254          Gen.GetCache().HeaderP->Dirty = true;
1255          SCacheF.Sync();
1256       }
1257
1258       // Build the status cache
1259       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1260                      Files.begin()+EndOfSource,Files.end()) == false)
1261          return false;
1262
1263       // FIXME: move me to a better place
1264       Gen.FinishCache(Progress);
1265    }
1266    if (Debug == true)
1267       std::clog << "Caches are ready for shipping" << std::endl;
1268
1269    if (_error->PendingError() == true)
1270       return false;
1271    if (OutMap != 0)
1272    {
1273       if (CacheF != 0)
1274       {
1275          delete Map.UnGuard();
1276          *OutMap = new MMap(*CacheF,0);
1277       }
1278       else
1279       {
1280          *OutMap = Map.UnGuard();
1281       }
1282    }
1283
1284    return true;
1285 }
1286                                                                         /*}}}*/
1287 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1288 // ---------------------------------------------------------------------
1289 /* */
1290 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1291    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1292 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1293 {
1294    vector<pkgIndexFile *> Files;
1295    unsigned long EndOfSource = Files.size();
1296    if (_system->AddStatusFiles(Files) == false)
1297       return false;
1298
1299    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1300    unsigned long CurrentSize = 0;
1301    unsigned long TotalSize = 0;
1302
1303    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1304
1305    // Build the status cache
1306    if (Progress != NULL)
1307       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1308    pkgCacheGenerator Gen(Map.Get(),Progress);
1309    if (_error->PendingError() == true)
1310       return false;
1311    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1312                   Files.begin()+EndOfSource,Files.end()) == false)
1313       return false;
1314
1315    // FIXME: move me to a better place
1316    Gen.FinishCache(Progress);
1317
1318    if (_error->PendingError() == true)
1319       return false;
1320    *OutMap = Map.UnGuard();
1321
1322    return true;
1323 }
1324                                                                         /*}}}*/