apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14 #include <config.h>
  15
  16 #include <apt-pkg/pkgcachegen.h>
  17 #include <apt-pkg/error.h>
  18 #include <apt-pkg/version.h>
  19 #include <apt-pkg/progress.h>
  20 #include <apt-pkg/sourcelist.h>
  21 #include <apt-pkg/configuration.h>
  22 #include <apt-pkg/aptconfiguration.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/sptr.h>
  25 #include <apt-pkg/pkgsystem.h>
  26 #include <apt-pkg/macros.h>
  27 #include <apt-pkg/tagfile.h>
  28 #include <apt-pkg/metaindex.h>
  29 #include <apt-pkg/fileutl.h>
  30
  31 #include <vector>
  32 #include <sys/stat.h>
  33 #include <unistd.h>
  34 #include <errno.h>
  35 #include <stdio.h>
  36
  37 #include <apti18n.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  41
  42 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* We set the dirty flag and make sure that is written to the disk */
  45 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  46                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  47                     FoundFileDeps(0)
  48 {
  49    CurrentFile = 0;
  50    memset(UniqHash,0,sizeof(UniqHash));
  51
  52    if (_error->PendingError() == true)
  53       return;
  54
  55    if (Map.Size() == 0)
  56    {
  57       // Setup the map interface..
  58       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  59       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  60          return;
  61
  62       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  63
  64       // Starting header
  65       *Cache.HeaderP = pkgCache::Header();
  66       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  67       Cache.HeaderP->VerSysName = idxVerSysName;
  68       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  69       Cache.HeaderP->Architecture = idxArchitecture;
  70       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  71          return;
  72       Cache.ReMap();
  73    }
  74    else
  75    {
  76       // Map directly from the existing file
  77       Cache.ReMap();
  78       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  79       if (Cache.VS != _system->VS)
  80       {
  81          _error->Error(_("Cache has an incompatible versioning system"));
  82          return;
  83       }
  84    }
  85
  86    Cache.HeaderP->Dirty = true;
  87    Map.Sync(0,sizeof(pkgCache::Header));
  88 }
  89                                                                         /*}}}*/
  90 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  91 // ---------------------------------------------------------------------
  92 /* We sync the data then unset the dirty flag in two steps so as to
  93    advoid a problem during a crash */
  94 pkgCacheGenerator::~pkgCacheGenerator()
  95 {
  96    if (_error->PendingError() == true)
  97       return;
  98    if (Map.Sync() == false)
  99       return;
 100
 101    Cache.HeaderP->Dirty = false;
 102    Cache.HeaderP->CacheFileSize = Map.Size();
 103    Map.Sync(0,sizeof(pkgCache::Header));
 104 }
 105                                                                         /*}}}*/
 106 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 107    if (oldMap == newMap)
 108       return;
 109
 110    if (_config->FindB("Debug::pkgCacheGen", false))
 111       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 112
 113    Cache.ReMap(false);
 114
 115    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 116
 117    for (size_t i = 0; i < _count(UniqHash); ++i)
 118       if (UniqHash[i] != 0)
 119          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 120
 121    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 122         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 123       (*i)->ReMap(oldMap, newMap);
 124    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 125         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 126       (*i)->ReMap(oldMap, newMap);
 127    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 128         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 129       (*i)->ReMap(oldMap, newMap);
 130    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 131         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 132       (*i)->ReMap(oldMap, newMap);
 133    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 134         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 135       (*i)->ReMap(oldMap, newMap);
 136    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 137         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 138       (*i)->ReMap(oldMap, newMap);
 139    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 140         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 141       (*i)->ReMap(oldMap, newMap);
 142 }                                                                       /*}}}*/
 143 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 144 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 145                                         const unsigned long &Len) {
 146    void const * const oldMap = Map.Data();
 147    map_ptrloc const index = Map.WriteString(String, Len);
 148    if (index != 0)
 149       ReMap(oldMap, Map.Data());
 150    return index;
 151 }
 152                                                                         /*}}}*/
 153 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 154 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 155    void const * const oldMap = Map.Data();
 156    map_ptrloc const index = Map.WriteString(String);
 157    if (index != 0)
 158       ReMap(oldMap, Map.Data());
 159    return index;
 160 }
 161                                                                         /*}}}*/
 162 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 163    void const * const oldMap = Map.Data();
 164    map_ptrloc const index = Map.Allocate(size);
 165    if (index != 0)
 166       ReMap(oldMap, Map.Data());
 167    return index;
 168 }
 169                                                                         /*}}}*/
 170 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 171 // ---------------------------------------------------------------------
 172 /* This provides the generation of the entries in the cache. Each loop
 173    goes through a single package record from the underlying parse engine. */
 174 bool pkgCacheGenerator::MergeList(ListParser &List,
 175                                   pkgCache::VerIterator *OutVer)
 176 {
 177    List.Owner = this;
 178
 179    unsigned int Counter = 0;
 180    while (List.Step() == true)
 181    {
 182       string const PackageName = List.Package();
 183       if (PackageName.empty() == true)
 184          return false;
 185
 186       string const Arch = List.Architecture();
 187
 188       // Get a pointer to the package structure
 189       pkgCache::PkgIterator Pkg;
 190       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 191       if (NewPackage(Pkg, PackageName, Arch) == false)
 192          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 193       Counter++;
 194       if (Counter % 100 == 0 && Progress != 0)
 195          Progress->Progress(List.Offset());
 196
 197       /* Get a pointer to the version structure. We know the list is sorted
 198          so we use that fact in the search. Insertion of new versions is
 199          done with correct sorting */
 200       string Version = List.Version();
 201       if (Version.empty() == true)
 202       {
 203          // we first process the package, then the descriptions
 204          // (this has the bonus that we get MMap error when we run out
 205          //  of MMap space)
 206          pkgCache::VerIterator Ver(Cache);
 207          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 208          if (List.UsePackage(Pkg, Ver) == false)
 209             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 210                                  PackageName.c_str());
 211
 212          // Find the right version to write the description
 213          MD5SumValue CurMd5 = List.Description_md5();
 214          Ver = Pkg.VersionList();
 215
 216          for (; Ver.end() == false; ++Ver)
 217          {
 218             pkgCache::DescIterator Desc = Ver.DescriptionList();
 219             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 220             map_ptrloc *LastDesc = &Ver->DescriptionList;
 221             bool duplicate=false;
 222
 223             // don't add a new description if we have one for the given
 224             // md5 && language
 225             for ( ; Desc.end() == false; ++Desc)
 226                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 227                    Desc.LanguageCode() == List.DescriptionLanguage())
 228                   duplicate=true;
 229             if(duplicate)
 230                continue;
 231
 232             for (Desc = Ver.DescriptionList();
 233                  Desc.end() == false;
 234                  LastDesc = &Desc->NextDesc, ++Desc)
 235             {
 236                if (MD5SumValue(Desc.md5()) == CurMd5)
 237                {
 238                   // Add new description
 239                   void const * const oldMap = Map.Data();
 240                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 241                   if (oldMap != Map.Data())
 242                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 243                   *LastDesc = descindex;
 244                   Desc->ParentPkg = Pkg.Index();
 245
 246                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 247                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 248                   break;
 249                }
 250             }
 251          }
 252
 253          continue;
 254       }
 255
 256       pkgCache::VerIterator Ver = Pkg.VersionList();
 257       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 258       map_ptrloc *LastVer = &Pkg->VersionList;
 259       void const * oldMap = Map.Data();
 260       int Res = 1;
 261       unsigned long const Hash = List.VersionHash();
 262       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 263       {
 264          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 265          // Version is higher as current version - insert here
 266          if (Res > 0)
 267             break;
 268          // Versionstrings are equal - is hash also equal?
 269          if (Res == 0 && Ver->Hash == Hash)
 270             break;
 271          // proceed with the next till we have either the right
 272          // or we found another version (which will be lower)
 273       }
 274
 275       /* We already have a version for this item, record that we saw it */
 276       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 277       {
 278          if (List.UsePackage(Pkg,Ver) == false)
 279             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 280                                  PackageName.c_str());
 281
 282          if (NewFileVer(Ver,List) == false)
 283             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 284                                  PackageName.c_str());
 285
 286          // Read only a single record and return
 287          if (OutVer != 0)
 288          {
 289             *OutVer = Ver;
 290             FoundFileDeps |= List.HasFileDeps();
 291             return true;
 292          }
 293
 294          continue;
 295       }
 296
 297       // Add a new version
 298       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 299       if (verindex == 0 && _error->PendingError())
 300          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 301                               PackageName.c_str(), 1);
 302
 303       if (oldMap != Map.Data())
 304          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 305       *LastVer = verindex;
 306       Ver->ParentPkg = Pkg.Index();
 307       Ver->Hash = Hash;
 308
 309       if (List.NewVersion(Ver) == false)
 310          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 311                               PackageName.c_str(), 2);
 312
 313       if (List.UsePackage(Pkg,Ver) == false)
 314          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 315                               PackageName.c_str());
 316
 317       if (NewFileVer(Ver,List) == false)
 318          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 319                               PackageName.c_str(), 3);
 320
 321       // Read only a single record and return
 322       if (OutVer != 0)
 323       {
 324          *OutVer = Ver;
 325          FoundFileDeps |= List.HasFileDeps();
 326          return true;
 327       }
 328
 329       /* Record the Description data. Description data always exist in
 330          Packages and Translation-* files. */
 331       pkgCache::DescIterator Desc = Ver.DescriptionList();
 332       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 333       map_ptrloc *LastDesc = &Ver->DescriptionList;
 334
 335       // Skip to the end of description set
 336       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 337
 338       // Add new description
 339       oldMap = Map.Data();
 340       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 341       if (oldMap != Map.Data())
 342          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 343       *LastDesc = descindex;
 344       Desc->ParentPkg = Pkg.Index();
 345
 346       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 347          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 348    }
 349
 350    FoundFileDeps |= List.HasFileDeps();
 351
 352    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 353       return _error->Error(_("Wow, you exceeded the number of package "
 354                              "names this APT is capable of."));
 355    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 356       return _error->Error(_("Wow, you exceeded the number of versions "
 357                              "this APT is capable of."));
 358    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 359       return _error->Error(_("Wow, you exceeded the number of descriptions "
 360                              "this APT is capable of."));
 361    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 362       return _error->Error(_("Wow, you exceeded the number of dependencies "
 363                              "this APT is capable of."));
 364    return true;
 365 }
 366                                                                         /*}}}*/
 367 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* If we found any file depends while parsing the main list we need to
 370    resolve them. Since it is undesired to load the entire list of files
 371    into the cache as virtual packages we do a two stage effort. MergeList
 372    identifies the file depends and this creates Provdies for them by
 373    re-parsing all the indexs. */
 374 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 375 {
 376    List.Owner = this;
 377
 378    unsigned int Counter = 0;
 379    while (List.Step() == true)
 380    {
 381       string PackageName = List.Package();
 382       if (PackageName.empty() == true)
 383          return false;
 384       string Version = List.Version();
 385       if (Version.empty() == true)
 386          continue;
 387
 388       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 389       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 390       if (Pkg.end() == true)
 391          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 392                                 PackageName.c_str());
 393       Counter++;
 394       if (Counter % 100 == 0 && Progress != 0)
 395          Progress->Progress(List.Offset());
 396
 397       unsigned long Hash = List.VersionHash();
 398       pkgCache::VerIterator Ver = Pkg.VersionList();
 399       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 400       for (; Ver.end() == false; ++Ver)
 401       {
 402          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 403          {
 404             if (List.CollectFileProvides(Cache,Ver) == false)
 405                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 406             break;
 407          }
 408       }
 409
 410       if (Ver.end() == true)
 411          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 412    }
 413
 414    return true;
 415 }
 416                                                                         /*}}}*/
 417 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 418 // ---------------------------------------------------------------------
 419 /* This creates a new group structure and adds it to the hash table */
 420 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 421 {
 422    Grp = Cache.FindGrp(Name);
 423    if (Grp.end() == false)
 424       return true;
 425
 426    // Get a structure
 427    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 428    if (unlikely(Group == 0))
 429       return false;
 430
 431    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 432    map_ptrloc const idxName = WriteStringInMap(Name);
 433    if (unlikely(idxName == 0))
 434       return false;
 435    Grp->Name = idxName;
 436
 437    // Insert it into the hash table
 438    unsigned long const Hash = Cache.Hash(Name);
 439    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 440    Cache.HeaderP->GrpHashTable[Hash] = Group;
 441
 442    Grp->ID = Cache.HeaderP->GroupCount++;
 443    return true;
 444 }
 445                                                                         /*}}}*/
 446 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* This creates a new package structure and adds it to the hash table */
 449 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 450                                         const string &Arch) {
 451    pkgCache::GrpIterator Grp;
 452    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 453    if (unlikely(NewGroup(Grp, Name) == false))
 454       return false;
 455
 456    Pkg = Grp.FindPkg(Arch);
 457       if (Pkg.end() == false)
 458          return true;
 459
 460    // Get a structure
 461    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 462    if (unlikely(Package == 0))
 463       return false;
 464    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 465
 466    // Insert the package into our package list
 467    if (Grp->FirstPackage == 0) // the group is new
 468    {
 469       // Insert it into the hash table
 470       unsigned long const Hash = Cache.Hash(Name);
 471       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 472       Cache.HeaderP->PkgHashTable[Hash] = Package;
 473       Grp->FirstPackage = Package;
 474    }
 475    else // Group the Packages together
 476    {
 477       // this package is the new last package
 478       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 479       Pkg->NextPackage = LastPkg->NextPackage;
 480       LastPkg->NextPackage = Package;
 481    }
 482    Grp->LastPackage = Package;
 483
 484    // Set the name, arch and the ID
 485    Pkg->Name = Grp->Name;
 486    Pkg->Group = Grp.Index();
 487    // all is mapped to the native architecture
 488    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 489    if (unlikely(idxArch == 0))
 490       return false;
 491    Pkg->Arch = idxArch;
 492    Pkg->ID = Cache.HeaderP->PackageCount++;
 493
 494    return true;
 495 }
 496                                                                         /*}}}*/
 497 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 498 // ---------------------------------------------------------------------
 499 /* */
 500 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 501                                    ListParser &List)
 502 {
 503    if (CurrentFile == 0)
 504       return true;
 505
 506    // Get a structure
 507    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 508    if (VerFile == 0)
 509       return 0;
 510
 511    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 512    VF->File = CurrentFile - Cache.PkgFileP;
 513
 514    // Link it to the end of the list
 515    map_ptrloc *Last = &Ver->FileList;
 516    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 517       Last = &V->NextFile;
 518    VF->NextFile = *Last;
 519    *Last = VF.Index();
 520
 521    VF->Offset = List.Offset();
 522    VF->Size = List.Size();
 523    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 524       Cache.HeaderP->MaxVerFileSize = VF->Size;
 525    Cache.HeaderP->VerFileCount++;
 526
 527    return true;
 528 }
 529                                                                         /*}}}*/
 530 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 531 // ---------------------------------------------------------------------
 532 /* This puts a version structure in the linked list */
 533 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 534                                             const string &VerStr,
 535                                             unsigned long Next)
 536 {
 537    // Get a structure
 538    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 539    if (Version == 0)
 540       return 0;
 541
 542    // Fill it in
 543    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 544    Ver->NextVer = Next;
 545    Ver->ID = Cache.HeaderP->VersionCount++;
 546    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 547    if (unlikely(idxVerStr == 0))
 548       return 0;
 549    Ver->VerStr = idxVerStr;
 550
 551    return Version;
 552 }
 553                                                                         /*}}}*/
 554 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 555 // ---------------------------------------------------------------------
 556 /* */
 557 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 558                                    ListParser &List)
 559 {
 560    if (CurrentFile == 0)
 561       return true;
 562
 563    // Get a structure
 564    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 565    if (DescFile == 0)
 566       return false;
 567
 568    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 569    DF->File = CurrentFile - Cache.PkgFileP;
 570
 571    // Link it to the end of the list
 572    map_ptrloc *Last = &Desc->FileList;
 573    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 574       Last = &D->NextFile;
 575
 576    DF->NextFile = *Last;
 577    *Last = DF.Index();
 578
 579    DF->Offset = List.Offset();
 580    DF->Size = List.Size();
 581    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 582       Cache.HeaderP->MaxDescFileSize = DF->Size;
 583    Cache.HeaderP->DescFileCount++;
 584
 585    return true;
 586 }
 587                                                                         /*}}}*/
 588 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 589 // ---------------------------------------------------------------------
 590 /* This puts a description structure in the linked list */
 591 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 592                                             const string &Lang,
 593                                             const MD5SumValue &md5sum,
 594                                             map_ptrloc Next)
 595 {
 596    // Get a structure
 597    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 598    if (Description == 0)
 599       return 0;
 600
 601    // Fill it in
 602    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 603    Desc->NextDesc = Next;
 604    Desc->ID = Cache.HeaderP->DescriptionCount++;
 605    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 606    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 607    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 608       return 0;
 609    Desc->language_code = idxlanguage_code;
 610    Desc->md5sum = idxmd5sum;
 611
 612    return Description;
 613 }
 614                                                                         /*}}}*/
 615 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 616 // ---------------------------------------------------------------------
 617 /* This prepares the Cache for delivery */
 618 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 619 {
 620    // FIXME: add progress reporting for this operation
 621    // Do we have different architectures in your groups ?
 622    vector<string> archs = APT::Configuration::getArchitectures();
 623    if (archs.size() > 1)
 624    {
 625       // Create Conflicts in between the group
 626       pkgCache::GrpIterator G = GetCache().GrpBegin();
 627       Dynamic<pkgCache::GrpIterator> DynG(G);
 628       for (; G.end() != true; ++G)
 629       {
 630          string const PkgName = G.Name();
 631          pkgCache::PkgIterator P = G.PackageList();
 632          Dynamic<pkgCache::PkgIterator> DynP(P);
 633          for (; P.end() != true; P = G.NextPkg(P))
 634          {
 635             pkgCache::PkgIterator allPkg;
 636             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 637             pkgCache::VerIterator V = P.VersionList();
 638             Dynamic<pkgCache::VerIterator> DynV(V);
 639             for (; V.end() != true; ++V)
 640             {
 641                // copy P.Arch() into a string here as a cache remap
 642                // in NewDepends() later may alter the pointer location
 643                string Arch = P.Arch() == NULL ? "" : P.Arch();
 644                map_ptrloc *OldDepLast = NULL;
 645                /* MultiArch handling introduces a lot of implicit Dependencies:
 646                 - MultiArch: same → Co-Installable if they have the same version
 647                 - Architecture: all → Need to be Co-Installable for internal reasons
 648                 - All others conflict with all other group members */
 649                bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 650                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 651                {
 652                   if (*A == Arch)
 653                      continue;
 654                   /* We allow only one installed arch at the time
 655                      per group, therefore each group member conflicts
 656                      with all other group members */
 657                   pkgCache::PkgIterator D = G.FindPkg(*A);
 658                   Dynamic<pkgCache::PkgIterator> DynD(D);
 659                   if (D.end() == true)
 660                      continue;
 661                   if (coInstall == true)
 662                   {
 663                      // Replaces: ${self}:other ( << ${binary:Version})
 664                      NewDepends(D, V, V.VerStr(),
 665                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 666                                 OldDepLast);
 667                      // Breaks: ${self}:other (!= ${binary:Version})
 668                      NewDepends(D, V, V.VerStr(),
 669                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 670                                 OldDepLast);
 671                   } else {
 672                         // Conflicts: ${self}:other
 673                         NewDepends(D, V, "",
 674                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 675                                    OldDepLast);
 676                   }
 677                }
 678             }
 679          }
 680       }
 681    }
 682    return true;
 683 }
 684                                                                         /*}}}*/
 685 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 686 // ---------------------------------------------------------------------
 687 /* This creates a dependency element in the tree. It is linked to the
 688    version and to the package that it is pointing to. */
 689 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 690                                    pkgCache::VerIterator &Ver,
 691                                    string const &Version,
 692                                    unsigned int const &Op,
 693                                    unsigned int const &Type,
 694                                    map_ptrloc* &OldDepLast)
 695 {
 696    void const * const oldMap = Map.Data();
 697    // Get a structure
 698    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 699    if (unlikely(Dependency == 0))
 700       return false;
 701
 702    // Fill it in
 703    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 704    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 705    Dep->ParentVer = Ver.Index();
 706    Dep->Type = Type;
 707    Dep->CompareOp = Op;
 708    Dep->ID = Cache.HeaderP->DependsCount++;
 709
 710    // Probe the reverse dependency list for a version string that matches
 711    if (Version.empty() == false)
 712    {
 713 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 714          if (I->Version != 0 && I.TargetVer() == Version)
 715             Dep->Version = I->Version;*/
 716       if (Dep->Version == 0) {
 717          map_ptrloc const index = WriteStringInMap(Version);
 718          if (unlikely(index == 0))
 719             return false;
 720          Dep->Version = index;
 721       }
 722    }
 723
 724    // Link it to the package
 725    Dep->Package = Pkg.Index();
 726    Dep->NextRevDepends = Pkg->RevDepends;
 727    Pkg->RevDepends = Dep.Index();
 728
 729    // Do we know where to link the Dependency to?
 730    if (OldDepLast == NULL)
 731    {
 732       OldDepLast = &Ver->DependsList;
 733       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 734          OldDepLast = &D->NextDepends;
 735    } else if (oldMap != Map.Data())
 736       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 737
 738    Dep->NextDepends = *OldDepLast;
 739    *OldDepLast = Dep.Index();
 740    OldDepLast = &Dep->NextDepends;
 741
 742    return true;
 743 }
 744                                                                         /*}}}*/
 745 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 746 // ---------------------------------------------------------------------
 747 /* This creates a Group and the Package to link this dependency to if
 748    needed and handles also the caching of the old endpoint */
 749 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 750                                                const string &PackageName,
 751                                                const string &Arch,
 752                                                const string &Version,
 753                                                unsigned int Op,
 754                                                unsigned int Type)
 755 {
 756    pkgCache::GrpIterator Grp;
 757    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 758    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 759       return false;
 760
 761    // Locate the target package
 762    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 763    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 764    if (Pkg.end() == true) {
 765       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 766          return false;
 767    }
 768
 769    // Is it a file dependency?
 770    if (unlikely(PackageName[0] == '/'))
 771       FoundFileDeps = true;
 772
 773    /* Caching the old end point speeds up generation substantially */
 774    if (OldDepVer != Ver) {
 775       OldDepLast = NULL;
 776       OldDepVer = Ver;
 777    }
 778
 779    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 780 }
 781                                                                         /*}}}*/
 782 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 783 // ---------------------------------------------------------------------
 784 /* */
 785 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 786                                                 const string &PkgName,
 787                                                 const string &PkgArch,
 788                                                 const string &Version)
 789 {
 790    pkgCache &Cache = Owner->Cache;
 791
 792    // We do not add self referencing provides
 793    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 794         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 795       return true;
 796
 797    // Get a structure
 798    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 799    if (unlikely(Provides == 0))
 800       return false;
 801    Cache.HeaderP->ProvidesCount++;
 802
 803    // Fill it in
 804    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 805    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 806    Prv->Version = Ver.Index();
 807    Prv->NextPkgProv = Ver->ProvidesList;
 808    Ver->ProvidesList = Prv.Index();
 809    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 810       return false;
 811
 812    // Locate the target package
 813    pkgCache::PkgIterator Pkg;
 814    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 815    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 816       return false;
 817
 818    // Link it to the package
 819    Prv->ParentPkg = Pkg.Index();
 820    Prv->NextProvides = Pkg->ProvidesList;
 821    Pkg->ProvidesList = Prv.Index();
 822
 823    return true;
 824 }
 825                                                                         /*}}}*/
 826 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 827 // ---------------------------------------------------------------------
 828 /* This is used to select which file is to be associated with all newly
 829    added versions. The caller is responsible for setting the IMS fields. */
 830 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 831                                    const pkgIndexFile &Index,
 832                                    unsigned long Flags)
 833 {
 834    // Get some space for the structure
 835    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 836    if (unlikely(idxFile == 0))
 837       return false;
 838    CurrentFile = Cache.PkgFileP + idxFile;
 839
 840    // Fill it in
 841    map_ptrloc const idxFileName = WriteStringInMap(File);
 842    map_ptrloc const idxSite = WriteUniqString(Site);
 843    if (unlikely(idxFileName == 0 || idxSite == 0))
 844       return false;
 845    CurrentFile->FileName = idxFileName;
 846    CurrentFile->Site = idxSite;
 847    CurrentFile->NextFile = Cache.HeaderP->FileList;
 848    CurrentFile->Flags = Flags;
 849    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 850    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 851    if (unlikely(idxIndexType == 0))
 852       return false;
 853    CurrentFile->IndexType = idxIndexType;
 854    PkgFileName = File;
 855    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 856    Cache.HeaderP->PackageFileCount++;
 857
 858    if (Progress != 0)
 859       Progress->SubProgress(Index.Size());
 860    return true;
 861 }
 862                                                                         /*}}}*/
 863 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 864 // ---------------------------------------------------------------------
 865 /* This is used to create handles to strings. Given the same text it
 866    always returns the same number */
 867 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 868                                                  unsigned int Size)
 869 {
 870    /* We use a very small transient hash table here, this speeds up generation
 871       by a fair amount on slower machines */
 872    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 873    if (Bucket != 0 &&
 874        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 875       return Bucket->String;
 876
 877    // Search for an insertion point
 878    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 879    int Res = 1;
 880    map_ptrloc *Last = &Cache.HeaderP->StringList;
 881    for (; I != Cache.StringItemP; Last = &I->NextItem,
 882         I = Cache.StringItemP + I->NextItem)
 883    {
 884       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 885       if (Res >= 0)
 886          break;
 887    }
 888
 889    // Match
 890    if (Res == 0)
 891    {
 892       Bucket = I;
 893       return I->String;
 894    }
 895
 896    // Get a structure
 897    void const * const oldMap = Map.Data();
 898    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 899    if (Item == 0)
 900       return 0;
 901
 902    map_ptrloc const idxString = WriteStringInMap(S,Size);
 903    if (unlikely(idxString == 0))
 904       return 0;
 905    if (oldMap != Map.Data()) {
 906       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 907       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 908    }
 909    *Last = Item;
 910
 911    // Fill in the structure
 912    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 913    ItemP->NextItem = I - Cache.StringItemP;
 914    ItemP->String = idxString;
 915
 916    Bucket = ItemP;
 917    return ItemP->String;
 918 }
 919                                                                         /*}}}*/
 920 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 921 // ---------------------------------------------------------------------
 922 /* This just verifies that each file in the list of index files exists,
 923    has matching attributes with the cache and the cache does not have
 924    any extra files. */
 925 static bool CheckValidity(const string &CacheFile,
 926                           pkgSourceList &List,
 927                           FileIterator Start,
 928                           FileIterator End,
 929                           MMap **OutMap = 0)
 930 {
 931    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 932    // No file, certainly invalid
 933    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 934    {
 935       if (Debug == true)
 936          std::clog << "CacheFile doesn't exist" << std::endl;
 937       return false;
 938    }
 939
 940    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
 941    {
 942       if (Debug == true)
 943          std::clog << "sources.list is newer than the cache" << std::endl;
 944       return false;
 945    }
 946
 947    // Map it
 948    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 949    SPtr<MMap> Map = new MMap(CacheF,0);
 950    pkgCache Cache(Map);
 951    if (_error->PendingError() == true || Map->Size() == 0)
 952    {
 953       if (Debug == true)
 954          std::clog << "Errors are pending or Map is empty()" << std::endl;
 955       _error->Discard();
 956       return false;
 957    }
 958
 959    /* Now we check every index file, see if it is in the cache,
 960       verify the IMS data and check that it is on the disk too.. */
 961    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 962    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 963    for (; Start != End; ++Start)
 964    {
 965       if (Debug == true)
 966          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 967       if ((*Start)->HasPackages() == false)
 968       {
 969          if (Debug == true)
 970             std::clog << "Has NO packages" << std::endl;
 971          continue;
 972       }
 973
 974       if ((*Start)->Exists() == false)
 975       {
 976 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 977          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 978                           (*Start)->Describe().c_str());
 979 #endif
 980          if (Debug == true)
 981             std::clog << "file doesn't exist" << std::endl;
 982          continue;
 983       }
 984
 985       // FindInCache is also expected to do an IMS check.
 986       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 987       if (File.end() == true)
 988       {
 989          if (Debug == true)
 990             std::clog << "FindInCache returned end-Pointer" << std::endl;
 991          return false;
 992       }
 993
 994       Visited[File->ID] = true;
 995       if (Debug == true)
 996          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 997    }
 998
 999    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1000       if (Visited[I] == false)
1001       {
1002          if (Debug == true)
1003             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1004          return false;
1005       }
1006
1007    if (_error->PendingError() == true)
1008    {
1009       if (Debug == true)
1010       {
1011          std::clog << "Validity failed because of pending errors:" << std::endl;
1012          _error->DumpErrors();
1013       }
1014       _error->Discard();
1015       return false;
1016    }
1017
1018    if (OutMap != 0)
1019       *OutMap = Map.UnGuard();
1020    return true;
1021 }
1022                                                                         /*}}}*/
1023 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1024 // ---------------------------------------------------------------------
1025 /* Size is kind of an abstract notion that is only used for the progress
1026    meter */
1027 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1028 {
1029    unsigned long TotalSize = 0;
1030    for (; Start != End; ++Start)
1031    {
1032       if ((*Start)->HasPackages() == false)
1033          continue;
1034       TotalSize += (*Start)->Size();
1035    }
1036    return TotalSize;
1037 }
1038                                                                         /*}}}*/
1039 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1040 // ---------------------------------------------------------------------
1041 /* */
1042 static bool BuildCache(pkgCacheGenerator &Gen,
1043                        OpProgress *Progress,
1044                        unsigned long &CurrentSize,unsigned long TotalSize,
1045                        FileIterator Start, FileIterator End)
1046 {
1047    FileIterator I;
1048    for (I = Start; I != End; ++I)
1049    {
1050       if ((*I)->HasPackages() == false)
1051          continue;
1052
1053       if ((*I)->Exists() == false)
1054          continue;
1055
1056       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1057       {
1058          _error->Warning("Duplicate sources.list entry %s",
1059                          (*I)->Describe().c_str());
1060          continue;
1061       }
1062
1063       unsigned long Size = (*I)->Size();
1064       if (Progress != NULL)
1065          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1066       CurrentSize += Size;
1067
1068       if ((*I)->Merge(Gen,Progress) == false)
1069          return false;
1070    }
1071
1072    if (Gen.HasFileDeps() == true)
1073    {
1074       if (Progress != NULL)
1075          Progress->Done();
1076       TotalSize = ComputeSize(Start, End);
1077       CurrentSize = 0;
1078       for (I = Start; I != End; ++I)
1079       {
1080          unsigned long Size = (*I)->Size();
1081          if (Progress != NULL)
1082             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1083          CurrentSize += Size;
1084          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1085             return false;
1086       }
1087    }
1088
1089    return true;
1090 }
1091                                                                         /*}}}*/
1092 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1093 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1094    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1095    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1096    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1097    Flags |= MMap::Moveable;
1098    if (_config->FindB("APT::Cache-Fallback", false) == true)
1099       Flags |= MMap::Fallback;
1100    if (CacheF != NULL)
1101       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1102    else
1103       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1104 }
1105                                                                         /*}}}*/
1106 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1107 // ---------------------------------------------------------------------
1108 /* This makes sure that the status cache (the cache that has all
1109    index files from the sources list and all local ones) is ready
1110    to be mmaped. If OutMap is not zero then a MMap object representing
1111    the cache will be stored there. This is pretty much mandetory if you
1112    are using AllowMem. AllowMem lets the function be run as non-root
1113    where it builds the cache 'fast' into a memory buffer. */
1114 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1115                         MMap **OutMap, bool AllowMem)
1116    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1117 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1118                         MMap **OutMap,bool AllowMem)
1119 {
1120    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1121
1122    vector<pkgIndexFile *> Files;
1123    for (vector<metaIndex *>::const_iterator i = List.begin();
1124         i != List.end();
1125         ++i)
1126    {
1127       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1128       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1129            j != Indexes->end();
1130            ++j)
1131          Files.push_back (*j);
1132    }
1133
1134    unsigned long const EndOfSource = Files.size();
1135    if (_system->AddStatusFiles(Files) == false)
1136       return false;
1137
1138    // Decide if we can write to the files..
1139    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1140    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1141
1142    // ensure the cache directory exists
1143    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1144    {
1145       string dir = _config->FindDir("Dir::Cache");
1146       size_t const len = dir.size();
1147       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1148          dir = dir.substr(0, len - 5);
1149       if (CacheFile.empty() == false)
1150          CreateDirectory(dir, flNotFile(CacheFile));
1151       if (SrcCacheFile.empty() == false)
1152          CreateDirectory(dir, flNotFile(SrcCacheFile));
1153    }
1154
1155    // Decide if we can write to the cache
1156    bool Writeable = false;
1157    if (CacheFile.empty() == false)
1158       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1159    else
1160       if (SrcCacheFile.empty() == false)
1161          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1162    if (Debug == true)
1163       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1164
1165    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1166       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1167
1168    if (Progress != NULL)
1169       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1170
1171    // Cache is OK, Fin.
1172    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1173    {
1174       if (Progress != NULL)
1175          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1176       if (Debug == true)
1177          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1178       return true;
1179    }
1180    else if (Debug == true)
1181          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1182
1183    /* At this point we know we need to reconstruct the package cache,
1184       begin. */
1185    SPtr<FileFd> CacheF;
1186    SPtr<DynamicMMap> Map;
1187    if (Writeable == true && CacheFile.empty() == false)
1188    {
1189       _error->PushToStack();
1190       unlink(CacheFile.c_str());
1191       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1192       fchmod(CacheF->Fd(),0644);
1193       Map = CreateDynamicMMap(CacheF, MMap::Public);
1194       if (_error->PendingError() == true)
1195       {
1196          delete CacheF.UnGuard();
1197          delete Map.UnGuard();
1198          if (Debug == true)
1199             std::clog << "Open filebased MMap FAILED" << std::endl;
1200          Writeable = false;
1201          if (AllowMem == false)
1202          {
1203             _error->MergeWithStack();
1204             return false;
1205          }
1206          _error->RevertToStack();
1207       }
1208       else if (Debug == true)
1209       {
1210          _error->MergeWithStack();
1211          std::clog << "Open filebased MMap" << std::endl;
1212       }
1213    }
1214    if (Writeable == false || CacheFile.empty() == true)
1215    {
1216       // Just build it in memory..
1217       Map = CreateDynamicMMap(NULL);
1218       if (Debug == true)
1219          std::clog << "Open memory Map (not filebased)" << std::endl;
1220    }
1221
1222    // Lets try the source cache.
1223    unsigned long CurrentSize = 0;
1224    unsigned long TotalSize = 0;
1225    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1226                      Files.begin()+EndOfSource) == true)
1227    {
1228       if (Debug == true)
1229          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1230       // Preload the map with the source cache
1231       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1232       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1233       if ((alloc == 0 && _error->PendingError())
1234                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1235                                 SCacheF.Size()) == false)
1236          return false;
1237
1238       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1239
1240       // Build the status cache
1241       pkgCacheGenerator Gen(Map.Get(),Progress);
1242       if (_error->PendingError() == true)
1243          return false;
1244       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1245                      Files.begin()+EndOfSource,Files.end()) == false)
1246          return false;
1247
1248       // FIXME: move me to a better place
1249       Gen.FinishCache(Progress);
1250    }
1251    else
1252    {
1253       if (Debug == true)
1254          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1255       TotalSize = ComputeSize(Files.begin(),Files.end());
1256
1257       // Build the source cache
1258       pkgCacheGenerator Gen(Map.Get(),Progress);
1259       if (_error->PendingError() == true)
1260          return false;
1261       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1262                      Files.begin(),Files.begin()+EndOfSource) == false)
1263          return false;
1264
1265       // Write it back
1266       if (Writeable == true && SrcCacheFile.empty() == false)
1267       {
1268          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1269          if (_error->PendingError() == true)
1270             return false;
1271
1272          fchmod(SCacheF.Fd(),0644);
1273
1274          // Write out the main data
1275          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1276             return _error->Error(_("IO Error saving source cache"));
1277          SCacheF.Sync();
1278
1279          // Write out the proper header
1280          Gen.GetCache().HeaderP->Dirty = false;
1281          if (SCacheF.Seek(0) == false ||
1282              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1283             return _error->Error(_("IO Error saving source cache"));
1284          Gen.GetCache().HeaderP->Dirty = true;
1285          SCacheF.Sync();
1286       }
1287
1288       // Build the status cache
1289       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1290                      Files.begin()+EndOfSource,Files.end()) == false)
1291          return false;
1292
1293       // FIXME: move me to a better place
1294       Gen.FinishCache(Progress);
1295    }
1296    if (Debug == true)
1297       std::clog << "Caches are ready for shipping" << std::endl;
1298
1299    if (_error->PendingError() == true)
1300       return false;
1301    if (OutMap != 0)
1302    {
1303       if (CacheF != 0)
1304       {
1305          delete Map.UnGuard();
1306          *OutMap = new MMap(*CacheF,0);
1307       }
1308       else
1309       {
1310          *OutMap = Map.UnGuard();
1311       }
1312    }
1313
1314    return true;
1315 }
1316                                                                         /*}}}*/
1317 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1318 // ---------------------------------------------------------------------
1319 /* */
1320 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1321    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1322 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1323 {
1324    vector<pkgIndexFile *> Files;
1325    unsigned long EndOfSource = Files.size();
1326    if (_system->AddStatusFiles(Files) == false)
1327       return false;
1328
1329    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1330    unsigned long CurrentSize = 0;
1331    unsigned long TotalSize = 0;
1332
1333    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1334
1335    // Build the status cache
1336    if (Progress != NULL)
1337       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1338    pkgCacheGenerator Gen(Map.Get(),Progress);
1339    if (_error->PendingError() == true)
1340       return false;
1341    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1342                   Files.begin()+EndOfSource,Files.end()) == false)
1343       return false;
1344
1345    // FIXME: move me to a better place
1346    Gen.FinishCache(Progress);
1347
1348    if (_error->PendingError() == true)
1349       return false;
1350    *OutMap = Map.UnGuard();
1351
1352    return true;
1353 }
1354                                                                         /*}}}*/