apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/metaindex.h>
  28 #include <apt-pkg/fileutl.h>
  29
  30 #include <vector>
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35
  36 #include <apti18n.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Cache.HeaderP->CacheFileSize = Map.Size();
 102    Map.Sync(0,sizeof(pkgCache::Header));
 103 }
 104                                                                         /*}}}*/
 105 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 106    if (oldMap == newMap)
 107       return;
 108
 109    if (_config->FindB("Debug::pkgCacheGen", false))
 110       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 111
 112    Cache.ReMap(false);
 113
 114    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 115
 116    for (size_t i = 0; i < _count(UniqHash); ++i)
 117       if (UniqHash[i] != 0)
 118          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 119
 120    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 121         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 122       (*i)->ReMap(oldMap, newMap);
 123    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 124         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 125       (*i)->ReMap(oldMap, newMap);
 126    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 127         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 128       (*i)->ReMap(oldMap, newMap);
 129    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 130         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 131       (*i)->ReMap(oldMap, newMap);
 132    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 133         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 134       (*i)->ReMap(oldMap, newMap);
 135    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 136         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 137       (*i)->ReMap(oldMap, newMap);
 138    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 139         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 140       (*i)->ReMap(oldMap, newMap);
 141 }                                                                       /*}}}*/
 142 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 143 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 144                                         const unsigned long &Len) {
 145    void const * const oldMap = Map.Data();
 146    map_ptrloc const index = Map.WriteString(String, Len);
 147    if (index != 0)
 148       ReMap(oldMap, Map.Data());
 149    return index;
 150 }
 151                                                                         /*}}}*/
 152 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 153 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 154    void const * const oldMap = Map.Data();
 155    map_ptrloc const index = Map.WriteString(String);
 156    if (index != 0)
 157       ReMap(oldMap, Map.Data());
 158    return index;
 159 }
 160                                                                         /*}}}*/
 161 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 162    void const * const oldMap = Map.Data();
 163    map_ptrloc const index = Map.Allocate(size);
 164    if (index != 0)
 165       ReMap(oldMap, Map.Data());
 166    return index;
 167 }
 168                                                                         /*}}}*/
 169 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 170 // ---------------------------------------------------------------------
 171 /* This provides the generation of the entries in the cache. Each loop
 172    goes through a single package record from the underlying parse engine. */
 173 bool pkgCacheGenerator::MergeList(ListParser &List,
 174                                   pkgCache::VerIterator *OutVer)
 175 {
 176    List.Owner = this;
 177
 178    unsigned int Counter = 0;
 179    while (List.Step() == true)
 180    {
 181       string const PackageName = List.Package();
 182       if (PackageName.empty() == true)
 183          return false;
 184
 185       string const Arch = List.Architecture();
 186
 187       // Get a pointer to the package structure
 188       pkgCache::PkgIterator Pkg;
 189       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 190       if (NewPackage(Pkg, PackageName, Arch) == false)
 191          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 192       Counter++;
 193       if (Counter % 100 == 0 && Progress != 0)
 194          Progress->Progress(List.Offset());
 195
 196       /* Get a pointer to the version structure. We know the list is sorted
 197          so we use that fact in the search. Insertion of new versions is
 198          done with correct sorting */
 199       string Version = List.Version();
 200       if (Version.empty() == true)
 201       {
 202          // we first process the package, then the descriptions
 203          // (this has the bonus that we get MMap error when we run out
 204          //  of MMap space)
 205          pkgCache::VerIterator Ver(Cache);
 206          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 207          if (List.UsePackage(Pkg, Ver) == false)
 208             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 209                                  PackageName.c_str());
 210
 211          // Find the right version to write the description
 212          MD5SumValue CurMd5 = List.Description_md5();
 213          Ver = Pkg.VersionList();
 214
 215          for (; Ver.end() == false; ++Ver)
 216          {
 217             pkgCache::DescIterator Desc = Ver.DescriptionList();
 218             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 219             map_ptrloc *LastDesc = &Ver->DescriptionList;
 220             bool duplicate=false;
 221
 222             // don't add a new description if we have one for the given
 223             // md5 && language
 224             for ( ; Desc.end() == false; ++Desc)
 225                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 226                    Desc.LanguageCode() == List.DescriptionLanguage())
 227                   duplicate=true;
 228             if(duplicate)
 229                continue;
 230
 231             for (Desc = Ver.DescriptionList();
 232                  Desc.end() == false;
 233                  LastDesc = &Desc->NextDesc, ++Desc)
 234             {
 235                if (MD5SumValue(Desc.md5()) == CurMd5)
 236                {
 237                   // Add new description
 238                   void const * const oldMap = Map.Data();
 239                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 240                   if (oldMap != Map.Data())
 241                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 242                   *LastDesc = descindex;
 243                   Desc->ParentPkg = Pkg.Index();
 244
 245                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 246                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 247                   break;
 248                }
 249             }
 250          }
 251
 252          continue;
 253       }
 254
 255       pkgCache::VerIterator Ver = Pkg.VersionList();
 256       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 257       map_ptrloc *LastVer = &Pkg->VersionList;
 258       void const * oldMap = Map.Data();
 259       int Res = 1;
 260       unsigned long const Hash = List.VersionHash();
 261       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 262       {
 263          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 264          // Version is higher as current version - insert here
 265          if (Res > 0)
 266             break;
 267          // Versionstrings are equal - is hash also equal?
 268          if (Res == 0 && Ver->Hash == Hash)
 269             break;
 270          // proceed with the next till we have either the right
 271          // or we found another version (which will be lower)
 272       }
 273
 274       /* We already have a version for this item, record that we saw it */
 275       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 276       {
 277          if (List.UsePackage(Pkg,Ver) == false)
 278             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 279                                  PackageName.c_str());
 280
 281          if (NewFileVer(Ver,List) == false)
 282             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 283                                  PackageName.c_str());
 284
 285          // Read only a single record and return
 286          if (OutVer != 0)
 287          {
 288             *OutVer = Ver;
 289             FoundFileDeps |= List.HasFileDeps();
 290             return true;
 291          }
 292
 293          continue;
 294       }
 295
 296       // Add a new version
 297       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 298       if (verindex == 0 && _error->PendingError())
 299          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 300                               PackageName.c_str(), 1);
 301
 302       if (oldMap != Map.Data())
 303          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 304       *LastVer = verindex;
 305       Ver->ParentPkg = Pkg.Index();
 306       Ver->Hash = Hash;
 307
 308       if (List.NewVersion(Ver) == false)
 309          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 310                               PackageName.c_str(), 2);
 311
 312       if (List.UsePackage(Pkg,Ver) == false)
 313          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 314                               PackageName.c_str());
 315
 316       if (NewFileVer(Ver,List) == false)
 317          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 318                               PackageName.c_str(), 3);
 319
 320       // Read only a single record and return
 321       if (OutVer != 0)
 322       {
 323          *OutVer = Ver;
 324          FoundFileDeps |= List.HasFileDeps();
 325          return true;
 326       }
 327
 328       /* Record the Description data. Description data always exist in
 329          Packages and Translation-* files. */
 330       pkgCache::DescIterator Desc = Ver.DescriptionList();
 331       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 332       map_ptrloc *LastDesc = &Ver->DescriptionList;
 333
 334       // Skip to the end of description set
 335       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 336
 337       // Add new description
 338       oldMap = Map.Data();
 339       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 340       if (oldMap != Map.Data())
 341          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 342       *LastDesc = descindex;
 343       Desc->ParentPkg = Pkg.Index();
 344
 345       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 346          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 347    }
 348
 349    FoundFileDeps |= List.HasFileDeps();
 350
 351    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 352       return _error->Error(_("Wow, you exceeded the number of package "
 353                              "names this APT is capable of."));
 354    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 355       return _error->Error(_("Wow, you exceeded the number of versions "
 356                              "this APT is capable of."));
 357    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 358       return _error->Error(_("Wow, you exceeded the number of descriptions "
 359                              "this APT is capable of."));
 360    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 361       return _error->Error(_("Wow, you exceeded the number of dependencies "
 362                              "this APT is capable of."));
 363    return true;
 364 }
 365                                                                         /*}}}*/
 366 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 367 // ---------------------------------------------------------------------
 368 /* If we found any file depends while parsing the main list we need to
 369    resolve them. Since it is undesired to load the entire list of files
 370    into the cache as virtual packages we do a two stage effort. MergeList
 371    identifies the file depends and this creates Provdies for them by
 372    re-parsing all the indexs. */
 373 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 374 {
 375    List.Owner = this;
 376
 377    unsigned int Counter = 0;
 378    while (List.Step() == true)
 379    {
 380       string PackageName = List.Package();
 381       if (PackageName.empty() == true)
 382          return false;
 383       string Version = List.Version();
 384       if (Version.empty() == true)
 385          continue;
 386
 387       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 388       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 389       if (Pkg.end() == true)
 390          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 391                                 PackageName.c_str());
 392       Counter++;
 393       if (Counter % 100 == 0 && Progress != 0)
 394          Progress->Progress(List.Offset());
 395
 396       unsigned long Hash = List.VersionHash();
 397       pkgCache::VerIterator Ver = Pkg.VersionList();
 398       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 399       for (; Ver.end() == false; ++Ver)
 400       {
 401          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 402          {
 403             if (List.CollectFileProvides(Cache,Ver) == false)
 404                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 405             break;
 406          }
 407       }
 408
 409       if (Ver.end() == true)
 410          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 411    }
 412
 413    return true;
 414 }
 415                                                                         /*}}}*/
 416 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 417 // ---------------------------------------------------------------------
 418 /* This creates a new group structure and adds it to the hash table */
 419 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 420 {
 421    Grp = Cache.FindGrp(Name);
 422    if (Grp.end() == false)
 423       return true;
 424
 425    // Get a structure
 426    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 427    if (unlikely(Group == 0))
 428       return false;
 429
 430    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 431    map_ptrloc const idxName = WriteStringInMap(Name);
 432    if (unlikely(idxName == 0))
 433       return false;
 434    Grp->Name = idxName;
 435
 436    // Insert it into the hash table
 437    unsigned long const Hash = Cache.Hash(Name);
 438    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 439    Cache.HeaderP->GrpHashTable[Hash] = Group;
 440
 441    Grp->ID = Cache.HeaderP->GroupCount++;
 442    return true;
 443 }
 444                                                                         /*}}}*/
 445 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 446 // ---------------------------------------------------------------------
 447 /* This creates a new package structure and adds it to the hash table */
 448 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 449                                         const string &Arch) {
 450    pkgCache::GrpIterator Grp;
 451    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 452    if (unlikely(NewGroup(Grp, Name) == false))
 453       return false;
 454
 455    Pkg = Grp.FindPkg(Arch);
 456       if (Pkg.end() == false)
 457          return true;
 458
 459    // Get a structure
 460    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 461    if (unlikely(Package == 0))
 462       return false;
 463    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 464
 465    // Insert the package into our package list
 466    if (Grp->FirstPackage == 0) // the group is new
 467    {
 468       // Insert it into the hash table
 469       unsigned long const Hash = Cache.Hash(Name);
 470       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 471       Cache.HeaderP->PkgHashTable[Hash] = Package;
 472       Grp->FirstPackage = Package;
 473    }
 474    else // Group the Packages together
 475    {
 476       // this package is the new last package
 477       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 478       Pkg->NextPackage = LastPkg->NextPackage;
 479       LastPkg->NextPackage = Package;
 480    }
 481    Grp->LastPackage = Package;
 482
 483    // Set the name, arch and the ID
 484    Pkg->Name = Grp->Name;
 485    Pkg->Group = Grp.Index();
 486    // all is mapped to the native architecture
 487    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 488    if (unlikely(idxArch == 0))
 489       return false;
 490    Pkg->Arch = idxArch;
 491    Pkg->ID = Cache.HeaderP->PackageCount++;
 492
 493    return true;
 494 }
 495                                                                         /*}}}*/
 496 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* */
 499 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 500                                    ListParser &List)
 501 {
 502    if (CurrentFile == 0)
 503       return true;
 504
 505    // Get a structure
 506    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 507    if (VerFile == 0)
 508       return 0;
 509
 510    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 511    VF->File = CurrentFile - Cache.PkgFileP;
 512
 513    // Link it to the end of the list
 514    map_ptrloc *Last = &Ver->FileList;
 515    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 516       Last = &V->NextFile;
 517    VF->NextFile = *Last;
 518    *Last = VF.Index();
 519
 520    VF->Offset = List.Offset();
 521    VF->Size = List.Size();
 522    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 523       Cache.HeaderP->MaxVerFileSize = VF->Size;
 524    Cache.HeaderP->VerFileCount++;
 525
 526    return true;
 527 }
 528                                                                         /*}}}*/
 529 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* This puts a version structure in the linked list */
 532 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 533                                             const string &VerStr,
 534                                             unsigned long Next)
 535 {
 536    // Get a structure
 537    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 538    if (Version == 0)
 539       return 0;
 540
 541    // Fill it in
 542    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 543    Ver->NextVer = Next;
 544    Ver->ID = Cache.HeaderP->VersionCount++;
 545    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 546    if (unlikely(idxVerStr == 0))
 547       return 0;
 548    Ver->VerStr = idxVerStr;
 549
 550    return Version;
 551 }
 552                                                                         /*}}}*/
 553 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 554 // ---------------------------------------------------------------------
 555 /* */
 556 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 557                                    ListParser &List)
 558 {
 559    if (CurrentFile == 0)
 560       return true;
 561
 562    // Get a structure
 563    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 564    if (DescFile == 0)
 565       return false;
 566
 567    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 568    DF->File = CurrentFile - Cache.PkgFileP;
 569
 570    // Link it to the end of the list
 571    map_ptrloc *Last = &Desc->FileList;
 572    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 573       Last = &D->NextFile;
 574
 575    DF->NextFile = *Last;
 576    *Last = DF.Index();
 577
 578    DF->Offset = List.Offset();
 579    DF->Size = List.Size();
 580    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 581       Cache.HeaderP->MaxDescFileSize = DF->Size;
 582    Cache.HeaderP->DescFileCount++;
 583
 584    return true;
 585 }
 586                                                                         /*}}}*/
 587 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 588 // ---------------------------------------------------------------------
 589 /* This puts a description structure in the linked list */
 590 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 591                                             const string &Lang,
 592                                             const MD5SumValue &md5sum,
 593                                             map_ptrloc Next)
 594 {
 595    // Get a structure
 596    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 597    if (Description == 0)
 598       return 0;
 599
 600    // Fill it in
 601    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 602    Desc->NextDesc = Next;
 603    Desc->ID = Cache.HeaderP->DescriptionCount++;
 604    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 605    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 606    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 607       return 0;
 608    Desc->language_code = idxlanguage_code;
 609    Desc->md5sum = idxmd5sum;
 610
 611    return Description;
 612 }
 613                                                                         /*}}}*/
 614 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 615 // ---------------------------------------------------------------------
 616 /* This prepares the Cache for delivery */
 617 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 618 {
 619    // FIXME: add progress reporting for this operation
 620    // Do we have different architectures in your groups ?
 621    vector<string> archs = APT::Configuration::getArchitectures();
 622    if (archs.size() > 1)
 623    {
 624       // Create Conflicts in between the group
 625       pkgCache::GrpIterator G = GetCache().GrpBegin();
 626       Dynamic<pkgCache::GrpIterator> DynG(G);
 627       for (; G.end() != true; ++G)
 628       {
 629          string const PkgName = G.Name();
 630          pkgCache::PkgIterator P = G.PackageList();
 631          Dynamic<pkgCache::PkgIterator> DynP(P);
 632          for (; P.end() != true; P = G.NextPkg(P))
 633          {
 634             pkgCache::PkgIterator allPkg;
 635             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 636             pkgCache::VerIterator V = P.VersionList();
 637             Dynamic<pkgCache::VerIterator> DynV(V);
 638             for (; V.end() != true; ++V)
 639             {
 640                // copy P.Arch() into a string here as a cache remap
 641                // in NewDepends() later may alter the pointer location
 642                string Arch = P.Arch() == NULL ? "" : P.Arch();
 643                map_ptrloc *OldDepLast = NULL;
 644                /* MultiArch handling introduces a lot of implicit Dependencies:
 645                 - MultiArch: same → Co-Installable if they have the same version
 646                 - Architecture: all → Need to be Co-Installable for internal reasons
 647                 - All others conflict with all other group members */
 648                bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 649                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 650                {
 651                   if (*A == Arch)
 652                      continue;
 653                   /* We allow only one installed arch at the time
 654                      per group, therefore each group member conflicts
 655                      with all other group members */
 656                   pkgCache::PkgIterator D = G.FindPkg(*A);
 657                   Dynamic<pkgCache::PkgIterator> DynD(D);
 658                   if (D.end() == true)
 659                      continue;
 660                   if (coInstall == true)
 661                   {
 662                      // Replaces: ${self}:other ( << ${binary:Version})
 663                      NewDepends(D, V, V.VerStr(),
 664                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 665                                 OldDepLast);
 666                      // Breaks: ${self}:other (!= ${binary:Version})
 667                      NewDepends(D, V, V.VerStr(),
 668                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 669                                 OldDepLast);
 670                   } else {
 671                         // Conflicts: ${self}:other
 672                         NewDepends(D, V, "",
 673                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 674                                    OldDepLast);
 675                   }
 676                }
 677             }
 678          }
 679       }
 680    }
 681    return true;
 682 }
 683                                                                         /*}}}*/
 684 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 685 // ---------------------------------------------------------------------
 686 /* This creates a dependency element in the tree. It is linked to the
 687    version and to the package that it is pointing to. */
 688 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 689                                    pkgCache::VerIterator &Ver,
 690                                    string const &Version,
 691                                    unsigned int const &Op,
 692                                    unsigned int const &Type,
 693                                    map_ptrloc* &OldDepLast)
 694 {
 695    void const * const oldMap = Map.Data();
 696    // Get a structure
 697    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 698    if (unlikely(Dependency == 0))
 699       return false;
 700
 701    // Fill it in
 702    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 703    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 704    Dep->ParentVer = Ver.Index();
 705    Dep->Type = Type;
 706    Dep->CompareOp = Op;
 707    Dep->ID = Cache.HeaderP->DependsCount++;
 708
 709    // Probe the reverse dependency list for a version string that matches
 710    if (Version.empty() == false)
 711    {
 712 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 713          if (I->Version != 0 && I.TargetVer() == Version)
 714             Dep->Version = I->Version;*/
 715       if (Dep->Version == 0) {
 716          map_ptrloc const index = WriteStringInMap(Version);
 717          if (unlikely(index == 0))
 718             return false;
 719          Dep->Version = index;
 720       }
 721    }
 722
 723    // Link it to the package
 724    Dep->Package = Pkg.Index();
 725    Dep->NextRevDepends = Pkg->RevDepends;
 726    Pkg->RevDepends = Dep.Index();
 727
 728    // Do we know where to link the Dependency to?
 729    if (OldDepLast == NULL)
 730    {
 731       OldDepLast = &Ver->DependsList;
 732       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 733          OldDepLast = &D->NextDepends;
 734    } else if (oldMap != Map.Data())
 735       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 736
 737    Dep->NextDepends = *OldDepLast;
 738    *OldDepLast = Dep.Index();
 739    OldDepLast = &Dep->NextDepends;
 740
 741    return true;
 742 }
 743                                                                         /*}}}*/
 744 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 745 // ---------------------------------------------------------------------
 746 /* This creates a Group and the Package to link this dependency to if
 747    needed and handles also the caching of the old endpoint */
 748 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 749                                                const string &PackageName,
 750                                                const string &Arch,
 751                                                const string &Version,
 752                                                unsigned int Op,
 753                                                unsigned int Type)
 754 {
 755    pkgCache::GrpIterator Grp;
 756    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 757    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 758       return false;
 759
 760    // Locate the target package
 761    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 762    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 763    if (Pkg.end() == true) {
 764       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 765          return false;
 766    }
 767
 768    // Is it a file dependency?
 769    if (unlikely(PackageName[0] == '/'))
 770       FoundFileDeps = true;
 771
 772    /* Caching the old end point speeds up generation substantially */
 773    if (OldDepVer != Ver) {
 774       OldDepLast = NULL;
 775       OldDepVer = Ver;
 776    }
 777
 778    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 779 }
 780                                                                         /*}}}*/
 781 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 782 // ---------------------------------------------------------------------
 783 /* */
 784 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 785                                                 const string &PkgName,
 786                                                 const string &PkgArch,
 787                                                 const string &Version)
 788 {
 789    pkgCache &Cache = Owner->Cache;
 790
 791    // We do not add self referencing provides
 792    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 793         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 794       return true;
 795
 796    // Get a structure
 797    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 798    if (unlikely(Provides == 0))
 799       return false;
 800    Cache.HeaderP->ProvidesCount++;
 801
 802    // Fill it in
 803    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 804    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 805    Prv->Version = Ver.Index();
 806    Prv->NextPkgProv = Ver->ProvidesList;
 807    Ver->ProvidesList = Prv.Index();
 808    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 809       return false;
 810
 811    // Locate the target package
 812    pkgCache::PkgIterator Pkg;
 813    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 814    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 815       return false;
 816
 817    // Link it to the package
 818    Prv->ParentPkg = Pkg.Index();
 819    Prv->NextProvides = Pkg->ProvidesList;
 820    Pkg->ProvidesList = Prv.Index();
 821
 822    return true;
 823 }
 824                                                                         /*}}}*/
 825 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 826 // ---------------------------------------------------------------------
 827 /* This is used to select which file is to be associated with all newly
 828    added versions. The caller is responsible for setting the IMS fields. */
 829 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 830                                    const pkgIndexFile &Index,
 831                                    unsigned long Flags)
 832 {
 833    // Get some space for the structure
 834    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 835    if (unlikely(idxFile == 0))
 836       return false;
 837    CurrentFile = Cache.PkgFileP + idxFile;
 838
 839    // Fill it in
 840    map_ptrloc const idxFileName = WriteStringInMap(File);
 841    map_ptrloc const idxSite = WriteUniqString(Site);
 842    if (unlikely(idxFileName == 0 || idxSite == 0))
 843       return false;
 844    CurrentFile->FileName = idxFileName;
 845    CurrentFile->Site = idxSite;
 846    CurrentFile->NextFile = Cache.HeaderP->FileList;
 847    CurrentFile->Flags = Flags;
 848    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 849    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 850    if (unlikely(idxIndexType == 0))
 851       return false;
 852    CurrentFile->IndexType = idxIndexType;
 853    PkgFileName = File;
 854    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 855    Cache.HeaderP->PackageFileCount++;
 856
 857    if (Progress != 0)
 858       Progress->SubProgress(Index.Size());
 859    return true;
 860 }
 861                                                                         /*}}}*/
 862 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 863 // ---------------------------------------------------------------------
 864 /* This is used to create handles to strings. Given the same text it
 865    always returns the same number */
 866 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 867                                                  unsigned int Size)
 868 {
 869    /* We use a very small transient hash table here, this speeds up generation
 870       by a fair amount on slower machines */
 871    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 872    if (Bucket != 0 &&
 873        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 874       return Bucket->String;
 875
 876    // Search for an insertion point
 877    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 878    int Res = 1;
 879    map_ptrloc *Last = &Cache.HeaderP->StringList;
 880    for (; I != Cache.StringItemP; Last = &I->NextItem,
 881         I = Cache.StringItemP + I->NextItem)
 882    {
 883       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 884       if (Res >= 0)
 885          break;
 886    }
 887
 888    // Match
 889    if (Res == 0)
 890    {
 891       Bucket = I;
 892       return I->String;
 893    }
 894
 895    // Get a structure
 896    void const * const oldMap = Map.Data();
 897    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 898    if (Item == 0)
 899       return 0;
 900
 901    map_ptrloc const idxString = WriteStringInMap(S,Size);
 902    if (unlikely(idxString == 0))
 903       return 0;
 904    if (oldMap != Map.Data()) {
 905       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 906       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 907    }
 908    *Last = Item;
 909
 910    // Fill in the structure
 911    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 912    ItemP->NextItem = I - Cache.StringItemP;
 913    ItemP->String = idxString;
 914
 915    Bucket = ItemP;
 916    return ItemP->String;
 917 }
 918                                                                         /*}}}*/
 919 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 920 // ---------------------------------------------------------------------
 921 /* This just verifies that each file in the list of index files exists,
 922    has matching attributes with the cache and the cache does not have
 923    any extra files. */
 924 static bool CheckValidity(const string &CacheFile,
 925                           pkgSourceList &List,
 926                           FileIterator Start,
 927                           FileIterator End,
 928                           MMap **OutMap = 0)
 929 {
 930    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 931    // No file, certainly invalid
 932    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 933    {
 934       if (Debug == true)
 935          std::clog << "CacheFile doesn't exist" << std::endl;
 936       return false;
 937    }
 938
 939    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
 940    {
 941       if (Debug == true)
 942          std::clog << "sources.list is newer than the cache" << std::endl;
 943       return false;
 944    }
 945
 946    // Map it
 947    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 948    SPtr<MMap> Map = new MMap(CacheF,0);
 949    pkgCache Cache(Map);
 950    if (_error->PendingError() == true || Map->Size() == 0)
 951    {
 952       if (Debug == true)
 953          std::clog << "Errors are pending or Map is empty()" << std::endl;
 954       _error->Discard();
 955       return false;
 956    }
 957
 958    /* Now we check every index file, see if it is in the cache,
 959       verify the IMS data and check that it is on the disk too.. */
 960    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 961    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 962    for (; Start != End; ++Start)
 963    {
 964       if (Debug == true)
 965          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 966       if ((*Start)->HasPackages() == false)
 967       {
 968          if (Debug == true)
 969             std::clog << "Has NO packages" << std::endl;
 970          continue;
 971       }
 972
 973       if ((*Start)->Exists() == false)
 974       {
 975 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 976          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 977                           (*Start)->Describe().c_str());
 978 #endif
 979          if (Debug == true)
 980             std::clog << "file doesn't exist" << std::endl;
 981          continue;
 982       }
 983
 984       // FindInCache is also expected to do an IMS check.
 985       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 986       if (File.end() == true)
 987       {
 988          if (Debug == true)
 989             std::clog << "FindInCache returned end-Pointer" << std::endl;
 990          return false;
 991       }
 992
 993       Visited[File->ID] = true;
 994       if (Debug == true)
 995          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 996    }
 997
 998    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 999       if (Visited[I] == false)
1000       {
1001          if (Debug == true)
1002             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1003          return false;
1004       }
1005
1006    if (_error->PendingError() == true)
1007    {
1008       if (Debug == true)
1009       {
1010          std::clog << "Validity failed because of pending errors:" << std::endl;
1011          _error->DumpErrors();
1012       }
1013       _error->Discard();
1014       return false;
1015    }
1016
1017    if (OutMap != 0)
1018       *OutMap = Map.UnGuard();
1019    return true;
1020 }
1021                                                                         /*}}}*/
1022 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1023 // ---------------------------------------------------------------------
1024 /* Size is kind of an abstract notion that is only used for the progress
1025    meter */
1026 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1027 {
1028    unsigned long TotalSize = 0;
1029    for (; Start != End; ++Start)
1030    {
1031       if ((*Start)->HasPackages() == false)
1032          continue;
1033       TotalSize += (*Start)->Size();
1034    }
1035    return TotalSize;
1036 }
1037                                                                         /*}}}*/
1038 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1039 // ---------------------------------------------------------------------
1040 /* */
1041 static bool BuildCache(pkgCacheGenerator &Gen,
1042                        OpProgress *Progress,
1043                        unsigned long &CurrentSize,unsigned long TotalSize,
1044                        FileIterator Start, FileIterator End)
1045 {
1046    FileIterator I;
1047    for (I = Start; I != End; ++I)
1048    {
1049       if ((*I)->HasPackages() == false)
1050          continue;
1051
1052       if ((*I)->Exists() == false)
1053          continue;
1054
1055       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1056       {
1057          _error->Warning("Duplicate sources.list entry %s",
1058                          (*I)->Describe().c_str());
1059          continue;
1060       }
1061
1062       unsigned long Size = (*I)->Size();
1063       if (Progress != NULL)
1064          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1065       CurrentSize += Size;
1066
1067       if ((*I)->Merge(Gen,Progress) == false)
1068          return false;
1069    }
1070
1071    if (Gen.HasFileDeps() == true)
1072    {
1073       if (Progress != NULL)
1074          Progress->Done();
1075       TotalSize = ComputeSize(Start, End);
1076       CurrentSize = 0;
1077       for (I = Start; I != End; ++I)
1078       {
1079          unsigned long Size = (*I)->Size();
1080          if (Progress != NULL)
1081             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1082          CurrentSize += Size;
1083          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1084             return false;
1085       }
1086    }
1087
1088    return true;
1089 }
1090                                                                         /*}}}*/
1091 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1092 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1093    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1094    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1095    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1096    Flags |= MMap::Moveable;
1097    if (_config->FindB("APT::Cache-Fallback", false) == true)
1098       Flags |= MMap::Fallback;
1099    if (CacheF != NULL)
1100       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1101    else
1102       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1103 }
1104                                                                         /*}}}*/
1105 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1106 // ---------------------------------------------------------------------
1107 /* This makes sure that the status cache (the cache that has all
1108    index files from the sources list and all local ones) is ready
1109    to be mmaped. If OutMap is not zero then a MMap object representing
1110    the cache will be stored there. This is pretty much mandetory if you
1111    are using AllowMem. AllowMem lets the function be run as non-root
1112    where it builds the cache 'fast' into a memory buffer. */
1113 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1114                         MMap **OutMap, bool AllowMem)
1115    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1116 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1117                         MMap **OutMap,bool AllowMem)
1118 {
1119    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1120
1121    vector<pkgIndexFile *> Files;
1122    for (vector<metaIndex *>::const_iterator i = List.begin();
1123         i != List.end();
1124         ++i)
1125    {
1126       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1127       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1128            j != Indexes->end();
1129            ++j)
1130          Files.push_back (*j);
1131    }
1132
1133    unsigned long const EndOfSource = Files.size();
1134    if (_system->AddStatusFiles(Files) == false)
1135       return false;
1136
1137    // Decide if we can write to the files..
1138    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1139    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1140
1141    // ensure the cache directory exists
1142    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1143    {
1144       string dir = _config->FindDir("Dir::Cache");
1145       size_t const len = dir.size();
1146       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1147          dir = dir.substr(0, len - 5);
1148       if (CacheFile.empty() == false)
1149          CreateDirectory(dir, flNotFile(CacheFile));
1150       if (SrcCacheFile.empty() == false)
1151          CreateDirectory(dir, flNotFile(SrcCacheFile));
1152    }
1153
1154    // Decide if we can write to the cache
1155    bool Writeable = false;
1156    if (CacheFile.empty() == false)
1157       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1158    else
1159       if (SrcCacheFile.empty() == false)
1160          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1161    if (Debug == true)
1162       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1163
1164    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1165       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1166
1167    if (Progress != NULL)
1168       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1169
1170    // Cache is OK, Fin.
1171    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1172    {
1173       if (Progress != NULL)
1174          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1175       if (Debug == true)
1176          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1177       return true;
1178    }
1179    else if (Debug == true)
1180          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1181
1182    /* At this point we know we need to reconstruct the package cache,
1183       begin. */
1184    SPtr<FileFd> CacheF;
1185    SPtr<DynamicMMap> Map;
1186    if (Writeable == true && CacheFile.empty() == false)
1187    {
1188       _error->PushToStack();
1189       unlink(CacheFile.c_str());
1190       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1191       fchmod(CacheF->Fd(),0644);
1192       Map = CreateDynamicMMap(CacheF, MMap::Public);
1193       if (_error->PendingError() == true)
1194       {
1195          delete CacheF.UnGuard();
1196          delete Map.UnGuard();
1197          if (Debug == true)
1198             std::clog << "Open filebased MMap FAILED" << std::endl;
1199          Writeable = false;
1200          if (AllowMem == false)
1201          {
1202             _error->MergeWithStack();
1203             return false;
1204          }
1205          _error->RevertToStack();
1206       }
1207       else if (Debug == true)
1208       {
1209          _error->MergeWithStack();
1210          std::clog << "Open filebased MMap" << std::endl;
1211       }
1212    }
1213    if (Writeable == false || CacheFile.empty() == true)
1214    {
1215       // Just build it in memory..
1216       Map = CreateDynamicMMap(NULL);
1217       if (Debug == true)
1218          std::clog << "Open memory Map (not filebased)" << std::endl;
1219    }
1220
1221    // Lets try the source cache.
1222    unsigned long CurrentSize = 0;
1223    unsigned long TotalSize = 0;
1224    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1225                      Files.begin()+EndOfSource) == true)
1226    {
1227       if (Debug == true)
1228          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1229       // Preload the map with the source cache
1230       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1231       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1232       if ((alloc == 0 && _error->PendingError())
1233                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1234                                 SCacheF.Size()) == false)
1235          return false;
1236
1237       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1238
1239       // Build the status cache
1240       pkgCacheGenerator Gen(Map.Get(),Progress);
1241       if (_error->PendingError() == true)
1242          return false;
1243       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1244                      Files.begin()+EndOfSource,Files.end()) == false)
1245          return false;
1246
1247       // FIXME: move me to a better place
1248       Gen.FinishCache(Progress);
1249    }
1250    else
1251    {
1252       if (Debug == true)
1253          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1254       TotalSize = ComputeSize(Files.begin(),Files.end());
1255
1256       // Build the source cache
1257       pkgCacheGenerator Gen(Map.Get(),Progress);
1258       if (_error->PendingError() == true)
1259          return false;
1260       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1261                      Files.begin(),Files.begin()+EndOfSource) == false)
1262          return false;
1263
1264       // Write it back
1265       if (Writeable == true && SrcCacheFile.empty() == false)
1266       {
1267          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1268          if (_error->PendingError() == true)
1269             return false;
1270
1271          fchmod(SCacheF.Fd(),0644);
1272
1273          // Write out the main data
1274          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1275             return _error->Error(_("IO Error saving source cache"));
1276          SCacheF.Sync();
1277
1278          // Write out the proper header
1279          Gen.GetCache().HeaderP->Dirty = false;
1280          if (SCacheF.Seek(0) == false ||
1281              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1282             return _error->Error(_("IO Error saving source cache"));
1283          Gen.GetCache().HeaderP->Dirty = true;
1284          SCacheF.Sync();
1285       }
1286
1287       // Build the status cache
1288       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1289                      Files.begin()+EndOfSource,Files.end()) == false)
1290          return false;
1291
1292       // FIXME: move me to a better place
1293       Gen.FinishCache(Progress);
1294    }
1295    if (Debug == true)
1296       std::clog << "Caches are ready for shipping" << std::endl;
1297
1298    if (_error->PendingError() == true)
1299       return false;
1300    if (OutMap != 0)
1301    {
1302       if (CacheF != 0)
1303       {
1304          delete Map.UnGuard();
1305          *OutMap = new MMap(*CacheF,0);
1306       }
1307       else
1308       {
1309          *OutMap = Map.UnGuard();
1310       }
1311    }
1312
1313    return true;
1314 }
1315                                                                         /*}}}*/
1316 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1317 // ---------------------------------------------------------------------
1318 /* */
1319 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1320    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1321 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1322 {
1323    vector<pkgIndexFile *> Files;
1324    unsigned long EndOfSource = Files.size();
1325    if (_system->AddStatusFiles(Files) == false)
1326       return false;
1327
1328    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1329    unsigned long CurrentSize = 0;
1330    unsigned long TotalSize = 0;
1331
1332    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1333
1334    // Build the status cache
1335    if (Progress != NULL)
1336       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1337    pkgCacheGenerator Gen(Map.Get(),Progress);
1338    if (_error->PendingError() == true)
1339       return false;
1340    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1341                   Files.begin()+EndOfSource,Files.end()) == false)
1342       return false;
1343
1344    // FIXME: move me to a better place
1345    Gen.FinishCache(Progress);
1346
1347    if (_error->PendingError() == true)
1348       return false;
1349    *OutMap = Map.UnGuard();
1350
1351    return true;
1352 }
1353                                                                         /*}}}*/