apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26
  27 #include <apt-pkg/tagfile.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::set<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Map.Sync(0,sizeof(pkgCache::Header));
 102 }
 103                                                                         /*}}}*/
 104 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 105    if (oldMap == newMap)
 106       return;
 107
 108    Cache.ReMap(false);
 109
 110    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 111
 112    for (size_t i = 0; i < _count(UniqHash); ++i)
 113       if (UniqHash[i] != 0)
 114          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 115
 116    for (std::set<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 117         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 118       (*i)->ReOwn(Cache, oldMap, newMap);
 119    for (std::set<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 120         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 121       (*i)->ReOwn(Cache, oldMap, newMap);
 122    for (std::set<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 123         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 124       (*i)->ReOwn(Cache, oldMap, newMap);
 125    for (std::set<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 126         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 127       (*i)->ReOwn(Cache, oldMap, newMap);
 128    for (std::set<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 129         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 130       (*i)->ReOwn(Cache, oldMap, newMap);
 131    for (std::set<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 132         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 133       (*i)->ReOwn(Cache, oldMap, newMap);
 134    for (std::set<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 135         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 136       (*i)->ReOwn(Cache, oldMap, newMap);
 137 }                                                                       /*}}}*/
 138 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 139 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 140                                         const unsigned long &Len) {
 141    void const * const oldMap = Map.Data();
 142    map_ptrloc const index = Map.WriteString(String, Len);
 143    if (index != 0)
 144       ReMap(oldMap, Map.Data());
 145    return index;
 146 }
 147                                                                         /*}}}*/
 148 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 149 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 150    void const * const oldMap = Map.Data();
 151    map_ptrloc const index = Map.WriteString(String);
 152    if (index != 0)
 153       ReMap(oldMap, Map.Data());
 154    return index;
 155 }
 156                                                                         /*}}}*/
 157 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 158    void const * const oldMap = Map.Data();
 159    map_ptrloc const index = Map.Allocate(size);
 160    if (index != 0)
 161       ReMap(oldMap, Map.Data());
 162    return index;
 163 }
 164                                                                         /*}}}*/
 165 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* This provides the generation of the entries in the cache. Each loop
 168    goes through a single package record from the underlying parse engine. */
 169 bool pkgCacheGenerator::MergeList(ListParser &List,
 170                                   pkgCache::VerIterator *OutVer)
 171 {
 172    List.Owner = this;
 173
 174    unsigned int Counter = 0;
 175    while (List.Step() == true)
 176    {
 177       string const PackageName = List.Package();
 178       if (PackageName.empty() == true)
 179          return false;
 180
 181       /* As we handle Arch all packages as architecture bounded
 182          we add all information to every (simulated) arch package */
 183       std::vector<string> genArch;
 184       if (List.ArchitectureAll() == true) {
 185          genArch = APT::Configuration::getArchitectures();
 186          if (genArch.size() != 1)
 187             genArch.push_back("all");
 188       } else
 189          genArch.push_back(List.Architecture());
 190
 191       for (std::vector<string>::const_iterator arch = genArch.begin();
 192            arch != genArch.end(); ++arch)
 193       {
 194       // Get a pointer to the package structure
 195       pkgCache::PkgIterator Pkg;
 196       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 197       if (NewPackage(Pkg, PackageName, *arch) == false)
 198          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 199       Counter++;
 200       if (Counter % 100 == 0 && Progress != 0)
 201          Progress->Progress(List.Offset());
 202
 203       /* Get a pointer to the version structure. We know the list is sorted
 204          so we use that fact in the search. Insertion of new versions is
 205          done with correct sorting */
 206       string Version = List.Version();
 207       if (Version.empty() == true)
 208       {
 209          // we first process the package, then the descriptions
 210          // (this has the bonus that we get MMap error when we run out
 211          //  of MMap space)
 212          pkgCache::VerIterator Ver(Cache);
 213          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 214          if (List.UsePackage(Pkg, Ver) == false)
 215             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 216                                  PackageName.c_str());
 217
 218          // Find the right version to write the description
 219          MD5SumValue CurMd5 = List.Description_md5();
 220          Ver = Pkg.VersionList();
 221
 222          for (; Ver.end() == false; ++Ver)
 223          {
 224             pkgCache::DescIterator Desc = Ver.DescriptionList();
 225             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 226             map_ptrloc *LastDesc = &Ver->DescriptionList;
 227             bool duplicate=false;
 228
 229             // don't add a new description if we have one for the given
 230             // md5 && language
 231             for ( ; Desc.end() == false; Desc++)
 232                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 233                    Desc.LanguageCode() == List.DescriptionLanguage())
 234                   duplicate=true;
 235             if(duplicate)
 236                continue;
 237
 238             for (Desc = Ver.DescriptionList();
 239                  Desc.end() == false;
 240                  LastDesc = &Desc->NextDesc, Desc++)
 241             {
 242                if (MD5SumValue(Desc.md5()) == CurMd5)
 243                {
 244                   // Add new description
 245                   void const * const oldMap = Map.Data();
 246                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 247                   if (oldMap != Map.Data())
 248                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 249                   *LastDesc = descindex;
 250                   Desc->ParentPkg = Pkg.Index();
 251
 252                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 253                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 254                   break;
 255                }
 256             }
 257          }
 258
 259          continue;
 260       }
 261
 262       pkgCache::VerIterator Ver = Pkg.VersionList();
 263       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 264       map_ptrloc *LastVer = &Pkg->VersionList;
 265       void const * oldMap = Map.Data();
 266       int Res = 1;
 267       unsigned long const Hash = List.VersionHash();
 268       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 269       {
 270          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 271          // Version is higher as current version - insert here
 272          if (Res > 0)
 273             break;
 274          // Versionstrings are equal - is hash also equal?
 275          if (Res == 0 && Ver->Hash == Hash)
 276             break;
 277          // proceed with the next till we have either the right
 278          // or we found another version (which will be lower)
 279       }
 280
 281       /* We already have a version for this item, record that we saw it */
 282       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 283       {
 284          if (List.UsePackage(Pkg,Ver) == false)
 285             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 286                                  PackageName.c_str());
 287
 288          if (NewFileVer(Ver,List) == false)
 289             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 290                                  PackageName.c_str());
 291
 292          // Read only a single record and return
 293          if (OutVer != 0)
 294          {
 295             *OutVer = Ver;
 296             FoundFileDeps |= List.HasFileDeps();
 297             return true;
 298          }
 299
 300          continue;
 301       }
 302
 303       // Add a new version
 304       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 305       if (verindex == 0 && _error->PendingError())
 306          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 307                               PackageName.c_str(), 1);
 308
 309       if (oldMap != Map.Data())
 310          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 311       *LastVer = verindex;
 312       Ver->ParentPkg = Pkg.Index();
 313       Ver->Hash = Hash;
 314
 315       if (List.NewVersion(Ver) == false)
 316          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 317                               PackageName.c_str(), 2);
 318
 319       if (List.UsePackage(Pkg,Ver) == false)
 320          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 321                               PackageName.c_str());
 322
 323       if (NewFileVer(Ver,List) == false)
 324          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 325                               PackageName.c_str(), 3);
 326
 327       // Read only a single record and return
 328       if (OutVer != 0)
 329       {
 330          *OutVer = Ver;
 331          FoundFileDeps |= List.HasFileDeps();
 332          return true;
 333       }
 334
 335       /* Record the Description data. Description data always exist in
 336          Packages and Translation-* files. */
 337       pkgCache::DescIterator Desc = Ver.DescriptionList();
 338       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 339       map_ptrloc *LastDesc = &Ver->DescriptionList;
 340
 341       // Skip to the end of description set
 342       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 343
 344       // Add new description
 345       oldMap = Map.Data();
 346       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 347       if (oldMap != Map.Data())
 348          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 349       *LastDesc = descindex;
 350       Desc->ParentPkg = Pkg.Index();
 351
 352       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 353          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 354       }
 355    }
 356
 357    FoundFileDeps |= List.HasFileDeps();
 358
 359    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 360       return _error->Error(_("Wow, you exceeded the number of package "
 361                              "names this APT is capable of."));
 362    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 363       return _error->Error(_("Wow, you exceeded the number of versions "
 364                              "this APT is capable of."));
 365    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 366       return _error->Error(_("Wow, you exceeded the number of descriptions "
 367                              "this APT is capable of."));
 368    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 369       return _error->Error(_("Wow, you exceeded the number of dependencies "
 370                              "this APT is capable of."));
 371    return true;
 372 }
 373                                                                         /*}}}*/
 374 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 375 // ---------------------------------------------------------------------
 376 /* If we found any file depends while parsing the main list we need to
 377    resolve them. Since it is undesired to load the entire list of files
 378    into the cache as virtual packages we do a two stage effort. MergeList
 379    identifies the file depends and this creates Provdies for them by
 380    re-parsing all the indexs. */
 381 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 382 {
 383    List.Owner = this;
 384
 385    unsigned int Counter = 0;
 386    while (List.Step() == true)
 387    {
 388       string PackageName = List.Package();
 389       if (PackageName.empty() == true)
 390          return false;
 391       string Version = List.Version();
 392       if (Version.empty() == true)
 393          continue;
 394
 395       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 396       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 397       if (Pkg.end() == true)
 398          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 399                                 PackageName.c_str());
 400       Counter++;
 401       if (Counter % 100 == 0 && Progress != 0)
 402          Progress->Progress(List.Offset());
 403
 404       unsigned long Hash = List.VersionHash();
 405       pkgCache::VerIterator Ver = Pkg.VersionList();
 406       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 407       for (; Ver.end() == false; Ver++)
 408       {
 409          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 410          {
 411             if (List.CollectFileProvides(Cache,Ver) == false)
 412                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 413             break;
 414          }
 415       }
 416
 417       if (Ver.end() == true)
 418          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 419    }
 420
 421    return true;
 422 }
 423                                                                         /*}}}*/
 424 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 425 // ---------------------------------------------------------------------
 426 /* This creates a new group structure and adds it to the hash table */
 427 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 428 {
 429    Grp = Cache.FindGrp(Name);
 430    if (Grp.end() == false)
 431       return true;
 432
 433    // Get a structure
 434    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 435    if (unlikely(Group == 0))
 436       return false;
 437
 438    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 439    map_ptrloc const idxName = WriteStringInMap(Name);
 440    if (unlikely(idxName == 0))
 441       return false;
 442    Grp->Name = idxName;
 443
 444    // Insert it into the hash table
 445    unsigned long const Hash = Cache.Hash(Name);
 446    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 447    Cache.HeaderP->GrpHashTable[Hash] = Group;
 448
 449    Grp->ID = Cache.HeaderP->GroupCount++;
 450    return true;
 451 }
 452                                                                         /*}}}*/
 453 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 454 // ---------------------------------------------------------------------
 455 /* This creates a new package structure and adds it to the hash table */
 456 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 457                                         const string &Arch) {
 458    pkgCache::GrpIterator Grp;
 459    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 460    if (unlikely(NewGroup(Grp, Name) == false))
 461       return false;
 462
 463    Pkg = Grp.FindPkg(Arch);
 464       if (Pkg.end() == false)
 465          return true;
 466
 467    // Get a structure
 468    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 469    if (unlikely(Package == 0))
 470       return false;
 471    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 472
 473    // Insert the package into our package list
 474    if (Grp->FirstPackage == 0) // the group is new
 475    {
 476       // Insert it into the hash table
 477       unsigned long const Hash = Cache.Hash(Name);
 478       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 479       Cache.HeaderP->PkgHashTable[Hash] = Package;
 480       Grp->FirstPackage = Package;
 481    }
 482    else // Group the Packages together
 483    {
 484       // this package is the new last package
 485       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 486       Pkg->NextPackage = LastPkg->NextPackage;
 487       LastPkg->NextPackage = Package;
 488    }
 489    Grp->LastPackage = Package;
 490
 491    // Set the name, arch and the ID
 492    Pkg->Name = Grp->Name;
 493    Pkg->Group = Grp.Index();
 494    map_ptrloc const idxArch = WriteUniqString(Arch.c_str());
 495    if (unlikely(idxArch == 0))
 496       return false;
 497    Pkg->Arch = idxArch;
 498    Pkg->ID = Cache.HeaderP->PackageCount++;
 499
 500    return true;
 501 }
 502                                                                         /*}}}*/
 503 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 504 // ---------------------------------------------------------------------
 505 /* */
 506 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 507                                    ListParser &List)
 508 {
 509    if (CurrentFile == 0)
 510       return true;
 511
 512    // Get a structure
 513    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 514    if (VerFile == 0)
 515       return 0;
 516
 517    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 518    VF->File = CurrentFile - Cache.PkgFileP;
 519
 520    // Link it to the end of the list
 521    map_ptrloc *Last = &Ver->FileList;
 522    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 523       Last = &V->NextFile;
 524    VF->NextFile = *Last;
 525    *Last = VF.Index();
 526
 527    VF->Offset = List.Offset();
 528    VF->Size = List.Size();
 529    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 530       Cache.HeaderP->MaxVerFileSize = VF->Size;
 531    Cache.HeaderP->VerFileCount++;
 532
 533    return true;
 534 }
 535                                                                         /*}}}*/
 536 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 537 // ---------------------------------------------------------------------
 538 /* This puts a version structure in the linked list */
 539 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 540                                             const string &VerStr,
 541                                             unsigned long Next)
 542 {
 543    // Get a structure
 544    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 545    if (Version == 0)
 546       return 0;
 547
 548    // Fill it in
 549    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 550    Ver->NextVer = Next;
 551    Ver->ID = Cache.HeaderP->VersionCount++;
 552    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 553    if (unlikely(idxVerStr == 0))
 554       return 0;
 555    Ver->VerStr = idxVerStr;
 556
 557    return Version;
 558 }
 559                                                                         /*}}}*/
 560 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 561 // ---------------------------------------------------------------------
 562 /* */
 563 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 564                                    ListParser &List)
 565 {
 566    if (CurrentFile == 0)
 567       return true;
 568
 569    // Get a structure
 570    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 571    if (DescFile == 0)
 572       return false;
 573
 574    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 575    DF->File = CurrentFile - Cache.PkgFileP;
 576
 577    // Link it to the end of the list
 578    map_ptrloc *Last = &Desc->FileList;
 579    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 580       Last = &D->NextFile;
 581
 582    DF->NextFile = *Last;
 583    *Last = DF.Index();
 584
 585    DF->Offset = List.Offset();
 586    DF->Size = List.Size();
 587    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 588       Cache.HeaderP->MaxDescFileSize = DF->Size;
 589    Cache.HeaderP->DescFileCount++;
 590
 591    return true;
 592 }
 593                                                                         /*}}}*/
 594 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 595 // ---------------------------------------------------------------------
 596 /* This puts a description structure in the linked list */
 597 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 598                                             const string &Lang,
 599                                             const MD5SumValue &md5sum,
 600                                             map_ptrloc Next)
 601 {
 602    // Get a structure
 603    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 604    if (Description == 0)
 605       return 0;
 606
 607    // Fill it in
 608    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 609    Desc->NextDesc = Next;
 610    Desc->ID = Cache.HeaderP->DescriptionCount++;
 611    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 612    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 613    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 614       return 0;
 615    Desc->language_code = idxlanguage_code;
 616    Desc->md5sum = idxmd5sum;
 617
 618    return Description;
 619 }
 620                                                                         /*}}}*/
 621 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* This prepares the Cache for delivery */
 624 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 625 {
 626    // FIXME: add progress reporting for this operation
 627    // Do we have different architectures in your groups ?
 628    vector<string> archs = APT::Configuration::getArchitectures();
 629    if (archs.size() > 1)
 630    {
 631       // Create Conflicts in between the group
 632       pkgCache::GrpIterator G = GetCache().GrpBegin();
 633       Dynamic<pkgCache::GrpIterator> DynG(G);
 634       for (; G.end() != true; G++)
 635       {
 636          string const PkgName = G.Name();
 637          pkgCache::PkgIterator P = G.PackageList();
 638          Dynamic<pkgCache::PkgIterator> DynP(P);
 639          for (; P.end() != true; P = G.NextPkg(P))
 640          {
 641             if (strcmp(P.Arch(),"all") == 0)
 642                continue;
 643             pkgCache::PkgIterator allPkg;
 644             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 645             pkgCache::VerIterator V = P.VersionList();
 646             Dynamic<pkgCache::VerIterator> DynV(V);
 647             for (; V.end() != true; V++)
 648             {
 649                string const Arch = V.Arch(true);
 650                map_ptrloc *OldDepLast = NULL;
 651                /* MultiArch handling introduces a lot of implicit Dependencies:
 652                 - MultiArch: same → Co-Installable if they have the same version
 653                 - Architecture: all → Need to be Co-Installable for internal reasons
 654                 - All others conflict with all other group members */
 655                bool const coInstall = (V->MultiArch == pkgCache::Version::All ||
 656                                         V->MultiArch == pkgCache::Version::Same);
 657                if (V->MultiArch == pkgCache::Version::All && allPkg.end() == true)
 658                   allPkg = G.FindPkg("all");
 659                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 660                {
 661                   if (*A == Arch)
 662                      continue;
 663                   /* We allow only one installed arch at the time
 664                      per group, therefore each group member conflicts
 665                      with all other group members */
 666                   pkgCache::PkgIterator D = G.FindPkg(*A);
 667                   Dynamic<pkgCache::PkgIterator> DynD(D);
 668                   if (D.end() == true)
 669                      continue;
 670                   if (coInstall == true)
 671                   {
 672                      // Replaces: ${self}:other ( << ${binary:Version})
 673                      NewDepends(D, V, V.VerStr(),
 674                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 675                                 OldDepLast);
 676                      // Breaks: ${self}:other (!= ${binary:Version})
 677                      NewDepends(D, V, V.VerStr(),
 678                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 679                                 OldDepLast);
 680                      if (V->MultiArch == pkgCache::Version::All)
 681                      {
 682                         // Depend on ${self}:all which does depend on nothing
 683                         NewDepends(allPkg, V, V.VerStr(),
 684                                    pkgCache::Dep::Equals, pkgCache::Dep::Depends,
 685                                    OldDepLast);
 686                      }
 687                   } else {
 688                         // Conflicts: ${self}:other
 689                         NewDepends(D, V, "",
 690                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 691                                    OldDepLast);
 692                   }
 693                }
 694             }
 695          }
 696       }
 697    }
 698    return true;
 699 }
 700                                                                         /*}}}*/
 701 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 702 // ---------------------------------------------------------------------
 703 /* This creates a dependency element in the tree. It is linked to the
 704    version and to the package that it is pointing to. */
 705 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 706                                    pkgCache::VerIterator &Ver,
 707                                    string const &Version,
 708                                    unsigned int const &Op,
 709                                    unsigned int const &Type,
 710                                    map_ptrloc *OldDepLast)
 711 {
 712    void const * const oldMap = Map.Data();
 713    // Get a structure
 714    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 715    if (unlikely(Dependency == 0))
 716       return false;
 717
 718    // Fill it in
 719    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 720    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 721    Dep->ParentVer = Ver.Index();
 722    Dep->Type = Type;
 723    Dep->CompareOp = Op;
 724    Dep->ID = Cache.HeaderP->DependsCount++;
 725
 726    // Probe the reverse dependency list for a version string that matches
 727    if (Version.empty() == false)
 728    {
 729 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 730          if (I->Version != 0 && I.TargetVer() == Version)
 731             Dep->Version = I->Version;*/
 732       if (Dep->Version == 0) {
 733          map_ptrloc const index = WriteStringInMap(Version);
 734          if (unlikely(index == 0))
 735             return false;
 736          Dep->Version = index;
 737       }
 738    }
 739
 740    // Link it to the package
 741    Dep->Package = Pkg.Index();
 742    Dep->NextRevDepends = Pkg->RevDepends;
 743    Pkg->RevDepends = Dep.Index();
 744
 745    // Do we know where to link the Dependency to?
 746    if (OldDepLast == NULL)
 747    {
 748       OldDepLast = &Ver->DependsList;
 749       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 750          OldDepLast = &D->NextDepends;
 751    } else if (oldMap != Map.Data())
 752       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 753
 754    Dep->NextDepends = *OldDepLast;
 755    *OldDepLast = Dep.Index();
 756    OldDepLast = &Dep->NextDepends;
 757
 758    return true;
 759 }
 760                                                                         /*}}}*/
 761 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 762 // ---------------------------------------------------------------------
 763 /* This creates a Group and the Package to link this dependency to if
 764    needed and handles also the caching of the old endpoint */
 765 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 766                                                const string &PackageName,
 767                                                const string &Arch,
 768                                                const string &Version,
 769                                                unsigned int Op,
 770                                                unsigned int Type)
 771 {
 772    pkgCache::GrpIterator Grp;
 773    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 774    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 775       return false;
 776
 777    // Locate the target package
 778    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 779    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 780    if (Pkg.end() == true) {
 781       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 782          return false;
 783    }
 784
 785    // Is it a file dependency?
 786    if (unlikely(PackageName[0] == '/'))
 787       FoundFileDeps = true;
 788
 789    /* Caching the old end point speeds up generation substantially */
 790    if (OldDepVer != Ver) {
 791       OldDepLast = NULL;
 792       OldDepVer = Ver;
 793    }
 794
 795    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 796 }
 797                                                                         /*}}}*/
 798 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 799 // ---------------------------------------------------------------------
 800 /* */
 801 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 802                                                 const string &PkgName,
 803                                                 const string &PkgArch,
 804                                                 const string &Version)
 805 {
 806    pkgCache &Cache = Owner->Cache;
 807
 808    // We do not add self referencing provides
 809    if (Ver.ParentPkg().Name() == PkgName && PkgArch == Ver.Arch(true))
 810       return true;
 811
 812    // Get a structure
 813    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 814    if (unlikely(Provides == 0))
 815       return false;
 816    Cache.HeaderP->ProvidesCount++;
 817
 818    // Fill it in
 819    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 820    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 821    Prv->Version = Ver.Index();
 822    Prv->NextPkgProv = Ver->ProvidesList;
 823    Ver->ProvidesList = Prv.Index();
 824    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 825       return false;
 826
 827    // Locate the target package
 828    pkgCache::PkgIterator Pkg;
 829    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 830    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 831       return false;
 832
 833    // Link it to the package
 834    Prv->ParentPkg = Pkg.Index();
 835    Prv->NextProvides = Pkg->ProvidesList;
 836    Pkg->ProvidesList = Prv.Index();
 837
 838    return true;
 839 }
 840                                                                         /*}}}*/
 841 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* This is used to select which file is to be associated with all newly
 844    added versions. The caller is responsible for setting the IMS fields. */
 845 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 846                                    const pkgIndexFile &Index,
 847                                    unsigned long Flags)
 848 {
 849    // Get some space for the structure
 850    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 851    if (unlikely(idxFile == 0))
 852       return false;
 853    CurrentFile = Cache.PkgFileP + idxFile;
 854
 855    // Fill it in
 856    map_ptrloc const idxFileName = WriteStringInMap(File);
 857    map_ptrloc const idxSite = WriteUniqString(Site);
 858    if (unlikely(idxFileName == 0 || idxSite == 0))
 859       return false;
 860    CurrentFile->FileName = idxFileName;
 861    CurrentFile->Site = idxSite;
 862    CurrentFile->NextFile = Cache.HeaderP->FileList;
 863    CurrentFile->Flags = Flags;
 864    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 865    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 866    if (unlikely(idxIndexType == 0))
 867       return false;
 868    CurrentFile->IndexType = idxIndexType;
 869    PkgFileName = File;
 870    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 871    Cache.HeaderP->PackageFileCount++;
 872
 873    if (Progress != 0)
 874       Progress->SubProgress(Index.Size());
 875    return true;
 876 }
 877                                                                         /*}}}*/
 878 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 879 // ---------------------------------------------------------------------
 880 /* This is used to create handles to strings. Given the same text it
 881    always returns the same number */
 882 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 883                                                  unsigned int Size)
 884 {
 885    /* We use a very small transient hash table here, this speeds up generation
 886       by a fair amount on slower machines */
 887    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 888    if (Bucket != 0 &&
 889        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 890       return Bucket->String;
 891
 892    // Search for an insertion point
 893    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 894    int Res = 1;
 895    map_ptrloc *Last = &Cache.HeaderP->StringList;
 896    for (; I != Cache.StringItemP; Last = &I->NextItem,
 897         I = Cache.StringItemP + I->NextItem)
 898    {
 899       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 900       if (Res >= 0)
 901          break;
 902    }
 903
 904    // Match
 905    if (Res == 0)
 906    {
 907       Bucket = I;
 908       return I->String;
 909    }
 910
 911    // Get a structure
 912    void const * const oldMap = Map.Data();
 913    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 914    if (Item == 0)
 915       return 0;
 916
 917    map_ptrloc const idxString = WriteStringInMap(S,Size);
 918    if (unlikely(idxString == 0))
 919       return 0;
 920    if (oldMap != Map.Data()) {
 921       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 922       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 923    }
 924    *Last = Item;
 925
 926    // Fill in the structure
 927    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 928    ItemP->NextItem = I - Cache.StringItemP;
 929    ItemP->String = idxString;
 930
 931    Bucket = ItemP;
 932    return ItemP->String;
 933 }
 934                                                                         /*}}}*/
 935 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 936 // ---------------------------------------------------------------------
 937 /* This just verifies that each file in the list of index files exists,
 938    has matching attributes with the cache and the cache does not have
 939    any extra files. */
 940 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 941                           FileIterator End,MMap **OutMap = 0)
 942 {
 943    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 944    // No file, certainly invalid
 945    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 946    {
 947       if (Debug == true)
 948          std::clog << "CacheFile doesn't exist" << std::endl;
 949       return false;
 950    }
 951
 952    // Map it
 953    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 954    SPtr<MMap> Map = new MMap(CacheF,0);
 955    pkgCache Cache(Map);
 956    if (_error->PendingError() == true || Map->Size() == 0)
 957    {
 958       if (Debug == true)
 959          std::clog << "Errors are pending or Map is empty()" << std::endl;
 960       _error->Discard();
 961       return false;
 962    }
 963
 964    /* Now we check every index file, see if it is in the cache,
 965       verify the IMS data and check that it is on the disk too.. */
 966    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 967    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 968    for (; Start != End; Start++)
 969    {
 970       if (Debug == true)
 971          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 972       if ((*Start)->HasPackages() == false)
 973       {
 974          if (Debug == true)
 975             std::clog << "Has NO packages" << std::endl;
 976          continue;
 977       }
 978
 979       if ((*Start)->Exists() == false)
 980       {
 981 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 982          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 983                           (*Start)->Describe().c_str());
 984 #endif
 985          if (Debug == true)
 986             std::clog << "file doesn't exist" << std::endl;
 987          continue;
 988       }
 989
 990       // FindInCache is also expected to do an IMS check.
 991       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 992       if (File.end() == true)
 993       {
 994          if (Debug == true)
 995             std::clog << "FindInCache returned end-Pointer" << std::endl;
 996          return false;
 997       }
 998
 999       Visited[File->ID] = true;
1000       if (Debug == true)
1001          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1002    }
1003
1004    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1005       if (Visited[I] == false)
1006       {
1007          if (Debug == true)
1008             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1009          return false;
1010       }
1011
1012    if (_error->PendingError() == true)
1013    {
1014       if (Debug == true)
1015       {
1016          std::clog << "Validity failed because of pending errors:" << std::endl;
1017          _error->DumpErrors();
1018       }
1019       _error->Discard();
1020       return false;
1021    }
1022
1023    if (OutMap != 0)
1024       *OutMap = Map.UnGuard();
1025    return true;
1026 }
1027                                                                         /*}}}*/
1028 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1029 // ---------------------------------------------------------------------
1030 /* Size is kind of an abstract notion that is only used for the progress
1031    meter */
1032 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1033 {
1034    unsigned long TotalSize = 0;
1035    for (; Start != End; Start++)
1036    {
1037       if ((*Start)->HasPackages() == false)
1038          continue;
1039       TotalSize += (*Start)->Size();
1040    }
1041    return TotalSize;
1042 }
1043                                                                         /*}}}*/
1044 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1045 // ---------------------------------------------------------------------
1046 /* */
1047 static bool BuildCache(pkgCacheGenerator &Gen,
1048                        OpProgress *Progress,
1049                        unsigned long &CurrentSize,unsigned long TotalSize,
1050                        FileIterator Start, FileIterator End)
1051 {
1052    FileIterator I;
1053    for (I = Start; I != End; I++)
1054    {
1055       if ((*I)->HasPackages() == false)
1056          continue;
1057
1058       if ((*I)->Exists() == false)
1059          continue;
1060
1061       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1062       {
1063          _error->Warning("Duplicate sources.list entry %s",
1064                          (*I)->Describe().c_str());
1065          continue;
1066       }
1067
1068       unsigned long Size = (*I)->Size();
1069       if (Progress != NULL)
1070          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1071       CurrentSize += Size;
1072
1073       if ((*I)->Merge(Gen,Progress) == false)
1074          return false;
1075    }
1076
1077    if (Gen.HasFileDeps() == true)
1078    {
1079       if (Progress != NULL)
1080          Progress->Done();
1081       TotalSize = ComputeSize(Start, End);
1082       CurrentSize = 0;
1083       for (I = Start; I != End; I++)
1084       {
1085          unsigned long Size = (*I)->Size();
1086          if (Progress != NULL)
1087             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1088          CurrentSize += Size;
1089          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1090             return false;
1091       }
1092    }
1093
1094    return true;
1095 }
1096                                                                         /*}}}*/
1097 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1098 // ---------------------------------------------------------------------
1099 /* This makes sure that the status cache (the cache that has all
1100    index files from the sources list and all local ones) is ready
1101    to be mmaped. If OutMap is not zero then a MMap object representing
1102    the cache will be stored there. This is pretty much mandetory if you
1103    are using AllowMem. AllowMem lets the function be run as non-root
1104    where it builds the cache 'fast' into a memory buffer. */
1105 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1106                         MMap **OutMap, bool AllowMem)
1107    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1108 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1109                         MMap **OutMap,bool AllowMem)
1110 {
1111    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1112    unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
1113
1114    vector<pkgIndexFile *> Files;
1115    for (vector<metaIndex *>::const_iterator i = List.begin();
1116         i != List.end();
1117         i++)
1118    {
1119       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1120       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1121            j != Indexes->end();
1122            j++)
1123          Files.push_back (*j);
1124    }
1125
1126    unsigned long const EndOfSource = Files.size();
1127    if (_system->AddStatusFiles(Files) == false)
1128       return false;
1129
1130    // Decide if we can write to the files..
1131    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1132    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1133
1134    // ensure the cache directory exists
1135    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1136    {
1137       string dir = _config->FindDir("Dir::Cache");
1138       size_t const len = dir.size();
1139       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1140          dir = dir.substr(0, len - 5);
1141       if (CacheFile.empty() == false)
1142          CreateDirectory(dir, flNotFile(CacheFile));
1143       if (SrcCacheFile.empty() == false)
1144          CreateDirectory(dir, flNotFile(SrcCacheFile));
1145    }
1146
1147    // Decide if we can write to the cache
1148    bool Writeable = false;
1149    if (CacheFile.empty() == false)
1150       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1151    else
1152       if (SrcCacheFile.empty() == false)
1153          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1154    if (Debug == true)
1155       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1156
1157    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1158       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1159
1160    if (Progress != NULL)
1161       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1162
1163    // Cache is OK, Fin.
1164    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
1165    {
1166       if (Progress != NULL)
1167          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1168       if (Debug == true)
1169          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1170       return true;
1171    }
1172    else if (Debug == true)
1173          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1174
1175    /* At this point we know we need to reconstruct the package cache,
1176       begin. */
1177    SPtr<FileFd> CacheF;
1178    SPtr<DynamicMMap> Map;
1179    if (Writeable == true && CacheFile.empty() == false)
1180    {
1181       unlink(CacheFile.c_str());
1182       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
1183       fchmod(CacheF->Fd(),0644);
1184       Map = new DynamicMMap(*CacheF,MMap::Public | MMap::Moveable, MapSize);
1185       if (_error->PendingError() == true)
1186          return false;
1187       if (Debug == true)
1188          std::clog << "Open filebased MMap" << std::endl;
1189    }
1190    else
1191    {
1192       // Just build it in memory..
1193       Map = new DynamicMMap(MMap::Moveable, MapSize);
1194       if (Debug == true)
1195          std::clog << "Open memory Map (not filebased)" << std::endl;
1196    }
1197
1198    // Lets try the source cache.
1199    unsigned long CurrentSize = 0;
1200    unsigned long TotalSize = 0;
1201    if (CheckValidity(SrcCacheFile,Files.begin(),
1202                      Files.begin()+EndOfSource) == true)
1203    {
1204       if (Debug == true)
1205          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1206       // Preload the map with the source cache
1207       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1208       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1209       if ((alloc == 0 && _error->PendingError())
1210                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1211                                 SCacheF.Size()) == false)
1212          return false;
1213
1214       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1215
1216       // Build the status cache
1217       pkgCacheGenerator Gen(Map.Get(),Progress);
1218       if (_error->PendingError() == true)
1219          return false;
1220       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1221                      Files.begin()+EndOfSource,Files.end()) == false)
1222          return false;
1223
1224       // FIXME: move me to a better place
1225       Gen.FinishCache(Progress);
1226    }
1227    else
1228    {
1229       if (Debug == true)
1230          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1231       TotalSize = ComputeSize(Files.begin(),Files.end());
1232
1233       // Build the source cache
1234       pkgCacheGenerator Gen(Map.Get(),Progress);
1235       if (_error->PendingError() == true)
1236          return false;
1237       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1238                      Files.begin(),Files.begin()+EndOfSource) == false)
1239          return false;
1240
1241       // Write it back
1242       if (Writeable == true && SrcCacheFile.empty() == false)
1243       {
1244          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
1245          if (_error->PendingError() == true)
1246             return false;
1247
1248          fchmod(SCacheF.Fd(),0644);
1249
1250          // Write out the main data
1251          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1252             return _error->Error(_("IO Error saving source cache"));
1253          SCacheF.Sync();
1254
1255          // Write out the proper header
1256          Gen.GetCache().HeaderP->Dirty = false;
1257          if (SCacheF.Seek(0) == false ||
1258              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1259             return _error->Error(_("IO Error saving source cache"));
1260          Gen.GetCache().HeaderP->Dirty = true;
1261          SCacheF.Sync();
1262       }
1263
1264       // Build the status cache
1265       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1266                      Files.begin()+EndOfSource,Files.end()) == false)
1267          return false;
1268
1269       // FIXME: move me to a better place
1270       Gen.FinishCache(Progress);
1271    }
1272    if (Debug == true)
1273       std::clog << "Caches are ready for shipping" << std::endl;
1274
1275    if (_error->PendingError() == true)
1276       return false;
1277    if (OutMap != 0)
1278    {
1279       if (CacheF != 0)
1280       {
1281          delete Map.UnGuard();
1282          *OutMap = new MMap(*CacheF,0);
1283       }
1284       else
1285       {
1286          *OutMap = Map.UnGuard();
1287       }
1288    }
1289
1290    return true;
1291 }
1292                                                                         /*}}}*/
1293 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1294 // ---------------------------------------------------------------------
1295 /* */
1296 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1297    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1298 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1299 {
1300    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
1301    vector<pkgIndexFile *> Files;
1302    unsigned long EndOfSource = Files.size();
1303    if (_system->AddStatusFiles(Files) == false)
1304       return false;
1305
1306    SPtr<DynamicMMap> Map = new DynamicMMap(MMap::Moveable, MapSize);
1307    unsigned long CurrentSize = 0;
1308    unsigned long TotalSize = 0;
1309
1310    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1311
1312    // Build the status cache
1313    if (Progress != NULL)
1314       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1315    pkgCacheGenerator Gen(Map.Get(),Progress);
1316    if (_error->PendingError() == true)
1317       return false;
1318    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1319                   Files.begin()+EndOfSource,Files.end()) == false)
1320       return false;
1321
1322    // FIXME: move me to a better place
1323    Gen.FinishCache(Progress);
1324
1325    if (_error->PendingError() == true)
1326       return false;
1327    *OutMap = Map.UnGuard();
1328
1329    return true;
1330 }
1331                                                                         /*}}}*/