apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26
  27 #include <apt-pkg/tagfile.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap(6);
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Map.Sync(0,sizeof(pkgCache::Header));
 102 }
 103                                                                         /*}}}*/
 104 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 105    if (oldMap == newMap)
 106       return;
 107
 108    Cache.ReMap(false);
 109
 110    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 111
 112    for (size_t i = 0; i < _count(UniqHash); ++i)
 113       if (UniqHash[i] != 0)
 114          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 115
 116    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 117         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 118       (*i)->ReOwn(Cache, oldMap, newMap);
 119    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 120         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 121       (*i)->ReOwn(Cache, oldMap, newMap);
 122    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 123         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 124       (*i)->ReOwn(Cache, oldMap, newMap);
 125    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 126         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 127       (*i)->ReOwn(Cache, oldMap, newMap);
 128    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 129         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 130       (*i)->ReOwn(Cache, oldMap, newMap);
 131    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 132         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 133       (*i)->ReOwn(Cache, oldMap, newMap);
 134    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 135         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 136       (*i)->ReOwn(Cache, oldMap, newMap);
 137 }                                                                       /*}}}*/
 138 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 139 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 140                                         const unsigned long &Len) {
 141    void const * const oldMap = Map.Data();
 142    map_ptrloc const index = Map.WriteString(String, Len);
 143    if (index != 0)
 144       ReMap(oldMap, Map.Data());
 145    return index;
 146 }
 147                                                                         /*}}}*/
 148 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 149 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 150    void const * const oldMap = Map.Data();
 151    map_ptrloc const index = Map.WriteString(String);
 152    if (index != 0)
 153       ReMap(oldMap, Map.Data());
 154    return index;
 155 }
 156                                                                         /*}}}*/
 157 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 158    void const * const oldMap = Map.Data();
 159    map_ptrloc const index = Map.Allocate(size);
 160    if (index != 0)
 161       ReMap(oldMap, Map.Data());
 162    return index;
 163 }
 164                                                                         /*}}}*/
 165 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* This provides the generation of the entries in the cache. Each loop
 168    goes through a single package record from the underlying parse engine. */
 169 bool pkgCacheGenerator::MergeList(ListParser &List,
 170                                   pkgCache::VerIterator *OutVer)
 171 {
 172    List.Owner = this;
 173
 174    unsigned int Counter = 0;
 175    while (List.Step() == true)
 176    {
 177       string const PackageName = List.Package();
 178       if (PackageName.empty() == true)
 179          return false;
 180
 181       /* As we handle Arch all packages as architecture bounded
 182          we add all information to every (simulated) arch package */
 183       std::vector<string> genArch;
 184       if (List.ArchitectureAll() == true) {
 185          genArch = APT::Configuration::getArchitectures();
 186          if (genArch.size() != 1)
 187             genArch.push_back("all");
 188       } else
 189          genArch.push_back(List.Architecture());
 190
 191       for (std::vector<string>::const_iterator arch = genArch.begin();
 192            arch != genArch.end(); ++arch)
 193       {
 194       // Get a pointer to the package structure
 195       pkgCache::PkgIterator Pkg;
 196       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 197       if (NewPackage(Pkg, PackageName, *arch) == false)
 198          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 199       Counter++;
 200       if (Counter % 100 == 0 && Progress != 0)
 201          Progress->Progress(List.Offset());
 202
 203       /* Get a pointer to the version structure. We know the list is sorted
 204          so we use that fact in the search. Insertion of new versions is
 205          done with correct sorting */
 206       string Version = List.Version();
 207       if (Version.empty() == true)
 208       {
 209          // we first process the package, then the descriptions
 210          // (this has the bonus that we get MMap error when we run out
 211          //  of MMap space)
 212          pkgCache::VerIterator Ver(Cache);
 213          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 214          if (List.UsePackage(Pkg, Ver) == false)
 215             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 216                                  PackageName.c_str());
 217
 218          // Find the right version to write the description
 219          MD5SumValue CurMd5 = List.Description_md5();
 220          Ver = Pkg.VersionList();
 221
 222          for (; Ver.end() == false; ++Ver)
 223          {
 224             pkgCache::DescIterator Desc = Ver.DescriptionList();
 225             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 226             map_ptrloc *LastDesc = &Ver->DescriptionList;
 227             bool duplicate=false;
 228
 229             // don't add a new description if we have one for the given
 230             // md5 && language
 231             for ( ; Desc.end() == false; Desc++)
 232                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 233                    Desc.LanguageCode() == List.DescriptionLanguage())
 234                   duplicate=true;
 235             if(duplicate)
 236                continue;
 237
 238             for (Desc = Ver.DescriptionList();
 239                  Desc.end() == false;
 240                  LastDesc = &Desc->NextDesc, Desc++)
 241             {
 242                if (MD5SumValue(Desc.md5()) == CurMd5)
 243                {
 244                   // Add new description
 245                   void const * const oldMap = Map.Data();
 246                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 247                   if (oldMap != Map.Data())
 248                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 249                   *LastDesc = descindex;
 250                   Desc->ParentPkg = Pkg.Index();
 251
 252                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 253                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 254                   break;
 255                }
 256             }
 257          }
 258
 259          continue;
 260       }
 261
 262       pkgCache::VerIterator Ver = Pkg.VersionList();
 263       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 264       map_ptrloc *LastVer = &Pkg->VersionList;
 265       void const * oldMap = Map.Data();
 266       int Res = 1;
 267       unsigned long const Hash = List.VersionHash();
 268       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 269       {
 270          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 271          // Version is higher as current version - insert here
 272          if (Res > 0)
 273             break;
 274          // Versionstrings are equal - is hash also equal?
 275          if (Res == 0 && Ver->Hash == Hash)
 276             break;
 277          // proceed with the next till we have either the right
 278          // or we found another version (which will be lower)
 279       }
 280
 281       /* We already have a version for this item, record that we saw it */
 282       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 283       {
 284          if (List.UsePackage(Pkg,Ver) == false)
 285             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 286                                  PackageName.c_str());
 287
 288          if (NewFileVer(Ver,List) == false)
 289             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 290                                  PackageName.c_str());
 291
 292          // Read only a single record and return
 293          if (OutVer != 0)
 294          {
 295             *OutVer = Ver;
 296             FoundFileDeps |= List.HasFileDeps();
 297             return true;
 298          }
 299
 300          continue;
 301       }
 302
 303       // Add a new version
 304       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 305       if (verindex == 0 && _error->PendingError())
 306          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 307                               PackageName.c_str(), 1);
 308
 309       if (oldMap != Map.Data())
 310          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 311       *LastVer = verindex;
 312       Ver->ParentPkg = Pkg.Index();
 313       Ver->Hash = Hash;
 314
 315       if (List.NewVersion(Ver) == false)
 316          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 317                               PackageName.c_str(), 2);
 318
 319       if (List.UsePackage(Pkg,Ver) == false)
 320          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 321                               PackageName.c_str());
 322
 323       if (NewFileVer(Ver,List) == false)
 324          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 325                               PackageName.c_str(), 3);
 326
 327       // Read only a single record and return
 328       if (OutVer != 0)
 329       {
 330          *OutVer = Ver;
 331          FoundFileDeps |= List.HasFileDeps();
 332          return true;
 333       }
 334
 335       /* Record the Description data. Description data always exist in
 336          Packages and Translation-* files. */
 337       pkgCache::DescIterator Desc = Ver.DescriptionList();
 338       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 339       map_ptrloc *LastDesc = &Ver->DescriptionList;
 340
 341       // Skip to the end of description set
 342       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 343
 344       // Add new description
 345       oldMap = Map.Data();
 346       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 347       if (oldMap != Map.Data())
 348          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 349       *LastDesc = descindex;
 350       Desc->ParentPkg = Pkg.Index();
 351
 352       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 353          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 354       }
 355    }
 356
 357    FoundFileDeps |= List.HasFileDeps();
 358
 359    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 360       return _error->Error(_("Wow, you exceeded the number of package "
 361                              "names this APT is capable of."));
 362    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 363       return _error->Error(_("Wow, you exceeded the number of versions "
 364                              "this APT is capable of."));
 365    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 366       return _error->Error(_("Wow, you exceeded the number of descriptions "
 367                              "this APT is capable of."));
 368    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 369       return _error->Error(_("Wow, you exceeded the number of dependencies "
 370                              "this APT is capable of."));
 371    return true;
 372 }
 373                                                                         /*}}}*/
 374 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 375 // ---------------------------------------------------------------------
 376 /* If we found any file depends while parsing the main list we need to
 377    resolve them. Since it is undesired to load the entire list of files
 378    into the cache as virtual packages we do a two stage effort. MergeList
 379    identifies the file depends and this creates Provdies for them by
 380    re-parsing all the indexs. */
 381 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 382 {
 383    List.Owner = this;
 384
 385    unsigned int Counter = 0;
 386    while (List.Step() == true)
 387    {
 388       string PackageName = List.Package();
 389       if (PackageName.empty() == true)
 390          return false;
 391       string Version = List.Version();
 392       if (Version.empty() == true)
 393          continue;
 394
 395       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 396       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 397       if (Pkg.end() == true)
 398          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 399                                 PackageName.c_str());
 400       Counter++;
 401       if (Counter % 100 == 0 && Progress != 0)
 402          Progress->Progress(List.Offset());
 403
 404       unsigned long Hash = List.VersionHash();
 405       pkgCache::VerIterator Ver = Pkg.VersionList();
 406       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 407       for (; Ver.end() == false; Ver++)
 408       {
 409          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 410          {
 411             if (List.CollectFileProvides(Cache,Ver) == false)
 412                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 413             break;
 414          }
 415       }
 416
 417       if (Ver.end() == true)
 418          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 419    }
 420
 421    return true;
 422 }
 423                                                                         /*}}}*/
 424 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 425 // ---------------------------------------------------------------------
 426 /* This creates a new group structure and adds it to the hash table */
 427 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 428 {
 429    Grp = Cache.FindGrp(Name);
 430    if (Grp.end() == false)
 431       return true;
 432
 433    // Get a structure
 434    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 435    if (unlikely(Group == 0))
 436       return false;
 437
 438    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 439    map_ptrloc const idxName = WriteStringInMap(Name);
 440    if (unlikely(idxName == 0))
 441       return false;
 442    Grp->Name = idxName;
 443
 444    // Insert it into the hash table
 445    unsigned long const Hash = Cache.Hash(Name);
 446    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 447    Cache.HeaderP->GrpHashTable[Hash] = Group;
 448
 449    Grp->ID = Cache.HeaderP->GroupCount++;
 450    return true;
 451 }
 452                                                                         /*}}}*/
 453 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 454 // ---------------------------------------------------------------------
 455 /* This creates a new package structure and adds it to the hash table */
 456 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 457                                         const string &Arch) {
 458    pkgCache::GrpIterator Grp;
 459    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 460    if (unlikely(NewGroup(Grp, Name) == false))
 461       return false;
 462
 463    Pkg = Grp.FindPkg(Arch);
 464       if (Pkg.end() == false)
 465          return true;
 466
 467    // Get a structure
 468    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 469    if (unlikely(Package == 0))
 470       return false;
 471    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 472
 473    // Insert the package into our package list
 474    if (Grp->FirstPackage == 0) // the group is new
 475    {
 476       // Insert it into the hash table
 477       unsigned long const Hash = Cache.Hash(Name);
 478       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 479       Cache.HeaderP->PkgHashTable[Hash] = Package;
 480       Grp->FirstPackage = Package;
 481    }
 482    else // Group the Packages together
 483    {
 484       // this package is the new last package
 485       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 486       Pkg->NextPackage = LastPkg->NextPackage;
 487       LastPkg->NextPackage = Package;
 488    }
 489    Grp->LastPackage = Package;
 490
 491    // Set the name, arch and the ID
 492    Pkg->Name = Grp->Name;
 493    Pkg->Group = Grp.Index();
 494    map_ptrloc const idxArch = WriteUniqString(Arch.c_str());
 495    if (unlikely(idxArch == 0))
 496       return false;
 497    Pkg->Arch = idxArch;
 498    Pkg->ID = Cache.HeaderP->PackageCount++;
 499
 500    return true;
 501 }
 502                                                                         /*}}}*/
 503 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 504 // ---------------------------------------------------------------------
 505 /* */
 506 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 507                                    ListParser &List)
 508 {
 509    if (CurrentFile == 0)
 510       return true;
 511
 512    // Get a structure
 513    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 514    if (VerFile == 0)
 515       return 0;
 516
 517    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 518    VF->File = CurrentFile - Cache.PkgFileP;
 519
 520    // Link it to the end of the list
 521    map_ptrloc *Last = &Ver->FileList;
 522    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 523       Last = &V->NextFile;
 524    VF->NextFile = *Last;
 525    *Last = VF.Index();
 526
 527    VF->Offset = List.Offset();
 528    VF->Size = List.Size();
 529    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 530       Cache.HeaderP->MaxVerFileSize = VF->Size;
 531    Cache.HeaderP->VerFileCount++;
 532
 533    return true;
 534 }
 535                                                                         /*}}}*/
 536 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 537 // ---------------------------------------------------------------------
 538 /* This puts a version structure in the linked list */
 539 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 540                                             const string &VerStr,
 541                                             unsigned long Next)
 542 {
 543    // Get a structure
 544    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 545    if (Version == 0)
 546       return 0;
 547
 548    // Fill it in
 549    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 550    Ver->NextVer = Next;
 551    Ver->ID = Cache.HeaderP->VersionCount++;
 552    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 553    if (unlikely(idxVerStr == 0))
 554       return 0;
 555    Ver->VerStr = idxVerStr;
 556
 557    return Version;
 558 }
 559                                                                         /*}}}*/
 560 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 561 // ---------------------------------------------------------------------
 562 /* */
 563 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 564                                    ListParser &List)
 565 {
 566    if (CurrentFile == 0)
 567       return true;
 568
 569    // Get a structure
 570    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 571    if (DescFile == 0)
 572       return false;
 573
 574    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 575    DF->File = CurrentFile - Cache.PkgFileP;
 576
 577    // Link it to the end of the list
 578    map_ptrloc *Last = &Desc->FileList;
 579    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 580       Last = &D->NextFile;
 581
 582    DF->NextFile = *Last;
 583    *Last = DF.Index();
 584
 585    DF->Offset = List.Offset();
 586    DF->Size = List.Size();
 587    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 588       Cache.HeaderP->MaxDescFileSize = DF->Size;
 589    Cache.HeaderP->DescFileCount++;
 590
 591    return true;
 592 }
 593                                                                         /*}}}*/
 594 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 595 // ---------------------------------------------------------------------
 596 /* This puts a description structure in the linked list */
 597 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 598                                             const string &Lang,
 599                                             const MD5SumValue &md5sum,
 600                                             map_ptrloc Next)
 601 {
 602    // Get a structure
 603    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 604    if (Description == 0)
 605       return 0;
 606
 607    // Fill it in
 608    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 609    Desc->NextDesc = Next;
 610    Desc->ID = Cache.HeaderP->DescriptionCount++;
 611    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 612    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 613    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 614       return 0;
 615    Desc->language_code = idxlanguage_code;
 616    Desc->md5sum = idxmd5sum;
 617
 618    return Description;
 619 }
 620                                                                         /*}}}*/
 621 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* This prepares the Cache for delivery */
 624 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 625 {
 626    // FIXME: add progress reporting for this operation
 627    // Do we have different architectures in your groups ?
 628    vector<string> archs = APT::Configuration::getArchitectures();
 629    if (archs.size() > 1)
 630    {
 631       // Create Conflicts in between the group
 632       pkgCache::GrpIterator G = GetCache().GrpBegin();
 633       Dynamic<pkgCache::GrpIterator> DynG(G);
 634       for (; G.end() != true; G++)
 635       {
 636          string const PkgName = G.Name();
 637          pkgCache::PkgIterator P = G.PackageList();
 638          Dynamic<pkgCache::PkgIterator> DynP(P);
 639          for (; P.end() != true; P = G.NextPkg(P))
 640          {
 641             if (strcmp(P.Arch(),"all") == 0)
 642                continue;
 643             pkgCache::PkgIterator allPkg;
 644             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 645             pkgCache::VerIterator V = P.VersionList();
 646             Dynamic<pkgCache::VerIterator> DynV(V);
 647             for (; V.end() != true; V++)
 648             {
 649                string const Arch = V.Arch(true);
 650                map_ptrloc *OldDepLast = NULL;
 651                /* MultiArch handling introduces a lot of implicit Dependencies:
 652                 - MultiArch: same → Co-Installable if they have the same version
 653                 - Architecture: all → Need to be Co-Installable for internal reasons
 654                 - All others conflict with all other group members */
 655                bool const coInstall = (V->MultiArch == pkgCache::Version::All ||
 656                                         V->MultiArch == pkgCache::Version::Same);
 657                if (V->MultiArch == pkgCache::Version::All && allPkg.end() == true)
 658                   allPkg = G.FindPkg("all");
 659                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 660                {
 661                   if (*A == Arch)
 662                      continue;
 663                   /* We allow only one installed arch at the time
 664                      per group, therefore each group member conflicts
 665                      with all other group members */
 666                   pkgCache::PkgIterator D = G.FindPkg(*A);
 667                   Dynamic<pkgCache::PkgIterator> DynD(D);
 668                   if (D.end() == true)
 669                      continue;
 670                   if (coInstall == true)
 671                   {
 672                      // Replaces: ${self}:other ( << ${binary:Version})
 673                      NewDepends(D, V, V.VerStr(),
 674                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 675                                 OldDepLast);
 676                      // Breaks: ${self}:other (!= ${binary:Version})
 677                      NewDepends(D, V, V.VerStr(),
 678                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 679                                 OldDepLast);
 680                      if (V->MultiArch == pkgCache::Version::All)
 681                      {
 682                         // Depend on ${self}:all which does depend on nothing
 683                         NewDepends(allPkg, V, V.VerStr(),
 684                                    pkgCache::Dep::Equals, pkgCache::Dep::Depends,
 685                                    OldDepLast);
 686                      }
 687                   } else {
 688                         // Conflicts: ${self}:other
 689                         NewDepends(D, V, "",
 690                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 691                                    OldDepLast);
 692                   }
 693                }
 694             }
 695          }
 696       }
 697    }
 698    return true;
 699 }
 700                                                                         /*}}}*/
 701 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 702 // ---------------------------------------------------------------------
 703 /* This creates a dependency element in the tree. It is linked to the
 704    version and to the package that it is pointing to. */
 705 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 706                                    pkgCache::VerIterator &Ver,
 707                                    string const &Version,
 708                                    unsigned int const &Op,
 709                                    unsigned int const &Type,
 710                                    map_ptrloc *OldDepLast)
 711 {
 712    void const * const oldMap = Map.Data();
 713    // Get a structure
 714    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 715    if (unlikely(Dependency == 0))
 716       return false;
 717
 718    // Fill it in
 719    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 720    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 721    Dep->ParentVer = Ver.Index();
 722    Dep->Type = Type;
 723    Dep->CompareOp = Op;
 724    Dep->ID = Cache.HeaderP->DependsCount++;
 725
 726    // Probe the reverse dependency list for a version string that matches
 727    if (Version.empty() == false)
 728    {
 729 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 730          if (I->Version != 0 && I.TargetVer() == Version)
 731             Dep->Version = I->Version;*/
 732       if (Dep->Version == 0) {
 733          map_ptrloc const index = WriteStringInMap(Version);
 734          if (unlikely(index == 0))
 735             return false;
 736          Dep->Version = index;
 737       }
 738    }
 739
 740    // Link it to the package
 741    Dep->Package = Pkg.Index();
 742    Dep->NextRevDepends = Pkg->RevDepends;
 743    Pkg->RevDepends = Dep.Index();
 744
 745    // Do we know where to link the Dependency to?
 746    if (OldDepLast == NULL)
 747    {
 748       OldDepLast = &Ver->DependsList;
 749       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 750          OldDepLast = &D->NextDepends;
 751    } else if (oldMap != Map.Data())
 752       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 753
 754    Dep->NextDepends = *OldDepLast;
 755    *OldDepLast = Dep.Index();
 756    OldDepLast = &Dep->NextDepends;
 757
 758    return true;
 759 }
 760                                                                         /*}}}*/
 761 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 762 // ---------------------------------------------------------------------
 763 /* This creates a Group and the Package to link this dependency to if
 764    needed and handles also the caching of the old endpoint */
 765 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 766                                                const string &PackageName,
 767                                                const string &Arch,
 768                                                const string &Version,
 769                                                unsigned int Op,
 770                                                unsigned int Type)
 771 {
 772    pkgCache::GrpIterator Grp;
 773    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 774    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 775       return false;
 776
 777    // Locate the target package
 778    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 779    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 780    if (Pkg.end() == true) {
 781       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 782          return false;
 783    }
 784
 785    // Is it a file dependency?
 786    if (unlikely(PackageName[0] == '/'))
 787       FoundFileDeps = true;
 788
 789    /* Caching the old end point speeds up generation substantially */
 790    if (OldDepVer != Ver) {
 791       OldDepLast = NULL;
 792       OldDepVer = Ver;
 793    }
 794
 795    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 796 }
 797                                                                         /*}}}*/
 798 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 799 // ---------------------------------------------------------------------
 800 /* */
 801 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 802                                                 const string &PkgName,
 803                                                 const string &PkgArch,
 804                                                 const string &Version)
 805 {
 806    pkgCache &Cache = Owner->Cache;
 807
 808    // We do not add self referencing provides
 809    if (Ver.ParentPkg().Name() == PkgName && PkgArch == Ver.Arch(true))
 810       return true;
 811
 812    // Get a structure
 813    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 814    if (unlikely(Provides == 0))
 815       return false;
 816    Cache.HeaderP->ProvidesCount++;
 817
 818    // Fill it in
 819    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 820    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 821    Prv->Version = Ver.Index();
 822    Prv->NextPkgProv = Ver->ProvidesList;
 823    Ver->ProvidesList = Prv.Index();
 824    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 825       return false;
 826
 827    // Locate the target package
 828    pkgCache::PkgIterator Pkg;
 829    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 830    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 831       return false;
 832
 833    // Link it to the package
 834    Prv->ParentPkg = Pkg.Index();
 835    Prv->NextProvides = Pkg->ProvidesList;
 836    Pkg->ProvidesList = Prv.Index();
 837
 838    return true;
 839 }
 840                                                                         /*}}}*/
 841 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* This is used to select which file is to be associated with all newly
 844    added versions. The caller is responsible for setting the IMS fields. */
 845 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 846                                    const pkgIndexFile &Index,
 847                                    unsigned long Flags)
 848 {
 849    // Get some space for the structure
 850    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 851    if (unlikely(idxFile == 0))
 852       return false;
 853    CurrentFile = Cache.PkgFileP + idxFile;
 854
 855    // Fill it in
 856    map_ptrloc const idxFileName = WriteStringInMap(File);
 857    map_ptrloc const idxSite = WriteUniqString(Site);
 858    if (unlikely(idxFileName == 0 || idxSite == 0))
 859       return false;
 860    CurrentFile->FileName = idxFileName;
 861    CurrentFile->Site = idxSite;
 862    CurrentFile->NextFile = Cache.HeaderP->FileList;
 863    CurrentFile->Flags = Flags;
 864    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 865    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 866    if (unlikely(idxIndexType == 0))
 867       return false;
 868    CurrentFile->IndexType = idxIndexType;
 869    PkgFileName = File;
 870    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 871    Cache.HeaderP->PackageFileCount++;
 872
 873    if (Progress != 0)
 874       Progress->SubProgress(Index.Size());
 875    return true;
 876 }
 877                                                                         /*}}}*/
 878 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 879 // ---------------------------------------------------------------------
 880 /* This is used to create handles to strings. Given the same text it
 881    always returns the same number */
 882 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 883                                                  unsigned int Size)
 884 {
 885    /* We use a very small transient hash table here, this speeds up generation
 886       by a fair amount on slower machines */
 887    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 888    if (Bucket != 0 &&
 889        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 890       return Bucket->String;
 891
 892    // Search for an insertion point
 893    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 894    int Res = 1;
 895    map_ptrloc *Last = &Cache.HeaderP->StringList;
 896    for (; I != Cache.StringItemP; Last = &I->NextItem,
 897         I = Cache.StringItemP + I->NextItem)
 898    {
 899       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 900       if (Res >= 0)
 901          break;
 902    }
 903
 904    // Match
 905    if (Res == 0)
 906    {
 907       Bucket = I;
 908       return I->String;
 909    }
 910
 911    // Get a structure
 912    void const * const oldMap = Map.Data();
 913    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 914    if (Item == 0)
 915       return 0;
 916
 917    map_ptrloc const idxString = WriteStringInMap(S,Size);
 918    if (unlikely(idxString == 0))
 919       return 0;
 920    if (oldMap != Map.Data()) {
 921       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 922       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 923    }
 924    *Last = Item;
 925
 926    // Fill in the structure
 927    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 928    ItemP->NextItem = I - Cache.StringItemP;
 929    ItemP->String = idxString;
 930
 931    Bucket = ItemP;
 932    return ItemP->String;
 933 }
 934                                                                         /*}}}*/
 935 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 936 // ---------------------------------------------------------------------
 937 /* This just verifies that each file in the list of index files exists,
 938    has matching attributes with the cache and the cache does not have
 939    any extra files. */
 940 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 941                           FileIterator End,MMap **OutMap = 0)
 942 {
 943    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 944    // No file, certainly invalid
 945    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 946    {
 947       if (Debug == true)
 948          std::clog << "CacheFile doesn't exist" << std::endl;
 949       return false;
 950    }
 951
 952    // Map it
 953    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 954    SPtr<MMap> Map = new MMap(CacheF,0);
 955    pkgCache Cache(Map);
 956    if (_error->PendingError() == true || Map->Size() == 0)
 957    {
 958       if (Debug == true)
 959          std::clog << "Errors are pending or Map is empty()" << std::endl;
 960       _error->Discard();
 961       return false;
 962    }
 963
 964    /* Now we check every index file, see if it is in the cache,
 965       verify the IMS data and check that it is on the disk too.. */
 966    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 967    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 968    for (; Start != End; Start++)
 969    {
 970       if (Debug == true)
 971          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 972       if ((*Start)->HasPackages() == false)
 973       {
 974          if (Debug == true)
 975             std::clog << "Has NO packages" << std::endl;
 976          continue;
 977       }
 978
 979       if ((*Start)->Exists() == false)
 980       {
 981 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 982          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 983                           (*Start)->Describe().c_str());
 984 #endif
 985          if (Debug == true)
 986             std::clog << "file doesn't exist" << std::endl;
 987          continue;
 988       }
 989
 990       // FindInCache is also expected to do an IMS check.
 991       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 992       if (File.end() == true)
 993       {
 994          if (Debug == true)
 995             std::clog << "FindInCache returned end-Pointer" << std::endl;
 996          return false;
 997       }
 998
 999       Visited[File->ID] = true;
1000       if (Debug == true)
1001          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1002    }
1003
1004    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1005       if (Visited[I] == false)
1006       {
1007          if (Debug == true)
1008             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1009          return false;
1010       }
1011
1012    if (_error->PendingError() == true)
1013    {
1014       if (Debug == true)
1015       {
1016          std::clog << "Validity failed because of pending errors:" << std::endl;
1017          _error->DumpErrors();
1018       }
1019       _error->Discard();
1020       return false;
1021    }
1022
1023    if (OutMap != 0)
1024       *OutMap = Map.UnGuard();
1025    return true;
1026 }
1027                                                                         /*}}}*/
1028 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1029 // ---------------------------------------------------------------------
1030 /* Size is kind of an abstract notion that is only used for the progress
1031    meter */
1032 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1033 {
1034    unsigned long TotalSize = 0;
1035    for (; Start != End; Start++)
1036    {
1037       if ((*Start)->HasPackages() == false)
1038          continue;
1039       TotalSize += (*Start)->Size();
1040    }
1041    return TotalSize;
1042 }
1043                                                                         /*}}}*/
1044 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1045 // ---------------------------------------------------------------------
1046 /* */
1047 static bool BuildCache(pkgCacheGenerator &Gen,
1048                        OpProgress *Progress,
1049                        unsigned long &CurrentSize,unsigned long TotalSize,
1050                        FileIterator Start, FileIterator End)
1051 {
1052    FileIterator I;
1053    for (I = Start; I != End; I++)
1054    {
1055       if ((*I)->HasPackages() == false)
1056          continue;
1057
1058       if ((*I)->Exists() == false)
1059          continue;
1060
1061       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1062       {
1063          _error->Warning("Duplicate sources.list entry %s",
1064                          (*I)->Describe().c_str());
1065          continue;
1066       }
1067
1068       unsigned long Size = (*I)->Size();
1069       if (Progress != NULL)
1070          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1071       CurrentSize += Size;
1072
1073       if ((*I)->Merge(Gen,Progress) == false)
1074          return false;
1075    }
1076
1077    if (Gen.HasFileDeps() == true)
1078    {
1079       if (Progress != NULL)
1080          Progress->Done();
1081       TotalSize = ComputeSize(Start, End);
1082       CurrentSize = 0;
1083       for (I = Start; I != End; I++)
1084       {
1085          unsigned long Size = (*I)->Size();
1086          if (Progress != NULL)
1087             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1088          CurrentSize += Size;
1089          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1090             return false;
1091       }
1092    }
1093
1094    return true;
1095 }
1096                                                                         /*}}}*/
1097 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1098    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1099    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1100    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1101    Flags |= MMap::Moveable;
1102    if (_config->FindB("APT::Cache-Fallback", false) == true)
1103       Flags |= MMap::Fallback;
1104    if (CacheF != NULL)
1105       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1106    else
1107       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1108 }
1109 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1110 // ---------------------------------------------------------------------
1111 /* This makes sure that the status cache (the cache that has all
1112    index files from the sources list and all local ones) is ready
1113    to be mmaped. If OutMap is not zero then a MMap object representing
1114    the cache will be stored there. This is pretty much mandetory if you
1115    are using AllowMem. AllowMem lets the function be run as non-root
1116    where it builds the cache 'fast' into a memory buffer. */
1117 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1118                         MMap **OutMap, bool AllowMem)
1119    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1120 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1121                         MMap **OutMap,bool AllowMem)
1122 {
1123    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1124
1125    vector<pkgIndexFile *> Files;
1126    for (vector<metaIndex *>::const_iterator i = List.begin();
1127         i != List.end();
1128         i++)
1129    {
1130       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1131       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1132            j != Indexes->end();
1133            j++)
1134          Files.push_back (*j);
1135    }
1136
1137    unsigned long const EndOfSource = Files.size();
1138    if (_system->AddStatusFiles(Files) == false)
1139       return false;
1140
1141    // Decide if we can write to the files..
1142    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1143    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1144
1145    // ensure the cache directory exists
1146    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1147    {
1148       string dir = _config->FindDir("Dir::Cache");
1149       size_t const len = dir.size();
1150       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1151          dir = dir.substr(0, len - 5);
1152       if (CacheFile.empty() == false)
1153          CreateDirectory(dir, flNotFile(CacheFile));
1154       if (SrcCacheFile.empty() == false)
1155          CreateDirectory(dir, flNotFile(SrcCacheFile));
1156    }
1157
1158    // Decide if we can write to the cache
1159    bool Writeable = false;
1160    if (CacheFile.empty() == false)
1161       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1162    else
1163       if (SrcCacheFile.empty() == false)
1164          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1165    if (Debug == true)
1166       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1167
1168    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1169       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1170
1171    if (Progress != NULL)
1172       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1173
1174    // Cache is OK, Fin.
1175    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
1176    {
1177       if (Progress != NULL)
1178          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1179       if (Debug == true)
1180          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1181       return true;
1182    }
1183    else if (Debug == true)
1184          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1185
1186    /* At this point we know we need to reconstruct the package cache,
1187       begin. */
1188    SPtr<FileFd> CacheF;
1189    SPtr<DynamicMMap> Map;
1190    if (Writeable == true && CacheFile.empty() == false)
1191    {
1192       unlink(CacheFile.c_str());
1193       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
1194       fchmod(CacheF->Fd(),0644);
1195       Map = CreateDynamicMMap(CacheF, MMap::Public);
1196       if (_error->PendingError() == true)
1197          return false;
1198       if (Debug == true)
1199          std::clog << "Open filebased MMap" << std::endl;
1200    }
1201    else
1202    {
1203       // Just build it in memory..
1204       Map = CreateDynamicMMap(NULL);
1205       if (Debug == true)
1206          std::clog << "Open memory Map (not filebased)" << std::endl;
1207    }
1208
1209    // Lets try the source cache.
1210    unsigned long CurrentSize = 0;
1211    unsigned long TotalSize = 0;
1212    if (CheckValidity(SrcCacheFile,Files.begin(),
1213                      Files.begin()+EndOfSource) == true)
1214    {
1215       if (Debug == true)
1216          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1217       // Preload the map with the source cache
1218       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1219       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1220       if ((alloc == 0 && _error->PendingError())
1221                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1222                                 SCacheF.Size()) == false)
1223          return false;
1224
1225       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1226
1227       // Build the status cache
1228       pkgCacheGenerator Gen(Map.Get(),Progress);
1229       if (_error->PendingError() == true)
1230          return false;
1231       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1232                      Files.begin()+EndOfSource,Files.end()) == false)
1233          return false;
1234
1235       // FIXME: move me to a better place
1236       Gen.FinishCache(Progress);
1237    }
1238    else
1239    {
1240       if (Debug == true)
1241          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1242       TotalSize = ComputeSize(Files.begin(),Files.end());
1243
1244       // Build the source cache
1245       pkgCacheGenerator Gen(Map.Get(),Progress);
1246       if (_error->PendingError() == true)
1247          return false;
1248       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1249                      Files.begin(),Files.begin()+EndOfSource) == false)
1250          return false;
1251
1252       // Write it back
1253       if (Writeable == true && SrcCacheFile.empty() == false)
1254       {
1255          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
1256          if (_error->PendingError() == true)
1257             return false;
1258
1259          fchmod(SCacheF.Fd(),0644);
1260
1261          // Write out the main data
1262          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1263             return _error->Error(_("IO Error saving source cache"));
1264          SCacheF.Sync();
1265
1266          // Write out the proper header
1267          Gen.GetCache().HeaderP->Dirty = false;
1268          if (SCacheF.Seek(0) == false ||
1269              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1270             return _error->Error(_("IO Error saving source cache"));
1271          Gen.GetCache().HeaderP->Dirty = true;
1272          SCacheF.Sync();
1273       }
1274
1275       // Build the status cache
1276       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1277                      Files.begin()+EndOfSource,Files.end()) == false)
1278          return false;
1279
1280       // FIXME: move me to a better place
1281       Gen.FinishCache(Progress);
1282    }
1283    if (Debug == true)
1284       std::clog << "Caches are ready for shipping" << std::endl;
1285
1286    if (_error->PendingError() == true)
1287       return false;
1288    if (OutMap != 0)
1289    {
1290       if (CacheF != 0)
1291       {
1292          delete Map.UnGuard();
1293          *OutMap = new MMap(*CacheF,0);
1294       }
1295       else
1296       {
1297          *OutMap = Map.UnGuard();
1298       }
1299    }
1300
1301    return true;
1302 }
1303                                                                         /*}}}*/
1304 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1305 // ---------------------------------------------------------------------
1306 /* */
1307 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1308    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1309 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1310 {
1311    vector<pkgIndexFile *> Files;
1312    unsigned long EndOfSource = Files.size();
1313    if (_system->AddStatusFiles(Files) == false)
1314       return false;
1315
1316    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1317    unsigned long CurrentSize = 0;
1318    unsigned long TotalSize = 0;
1319
1320    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1321
1322    // Build the status cache
1323    if (Progress != NULL)
1324       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1325    pkgCacheGenerator Gen(Map.Get(),Progress);
1326    if (_error->PendingError() == true)
1327       return false;
1328    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1329                   Files.begin()+EndOfSource,Files.end()) == false)
1330       return false;
1331
1332    // FIXME: move me to a better place
1333    Gen.FinishCache(Progress);
1334
1335    if (_error->PendingError() == true)
1336       return false;
1337    *OutMap = Map.UnGuard();
1338
1339    return true;
1340 }
1341                                                                         /*}}}*/