apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.28 1999/02/08 07:30:49 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25
  26 #include <sys/stat.h>
  27 #include <unistd.h>
  28                                                                         /*}}}*/
  29
  30 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  31 // ---------------------------------------------------------------------
  32 /* We set the diry flag and make sure that is written to the disk */
  33 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  34                     Map(Map), Cache(Map), Progress(Prog)
  35 {
  36    if (_error->PendingError() == true)
  37       return;
  38
  39    if (Map.Size() == 0)
  40    {
  41       Map.RawAllocate(sizeof(pkgCache::Header));
  42       *Cache.HeaderP = pkgCache::Header();
  43    }
  44    Cache.HeaderP->Dirty = true;
  45    Map.Sync(0,sizeof(pkgCache::Header));
  46    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  47 }
  48                                                                         /*}}}*/
  49 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  50 // ---------------------------------------------------------------------
  51 /* We sync the data then unset the dirty flag in two steps so as to
  52    advoid a problem during a crash */
  53 pkgCacheGenerator::~pkgCacheGenerator()
  54 {
  55    if (_error->PendingError() == true)
  56       return;
  57    if (Map.Sync() == false)
  58       return;
  59
  60    Cache.HeaderP->Dirty = false;
  61    Map.Sync(0,sizeof(pkgCache::Header));
  62 }
  63                                                                         /*}}}*/
  64 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* This provides the generation of the entries in the cache. Each loop
  67    goes through a single package record from the underlying parse engine. */
  68 bool pkgCacheGenerator::MergeList(ListParser &List)
  69 {
  70    List.Owner = this;
  71
  72    int Counter = 0;
  73    while (List.Step() == true)
  74    {
  75       // Get a pointer to the package structure
  76       string PackageName = List.Package();
  77       if (PackageName.empty() == true)
  78          return false;
  79
  80       pkgCache::PkgIterator Pkg;
  81       if (NewPackage(Pkg,PackageName) == false)
  82          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  83       Counter++;
  84       if (Counter % 100 == 0)
  85          Progress.Progress(List.Offset());
  86
  87       /* Get a pointer to the version structure. We know the list is sorted
  88          so we use that fact in the search. Insertion of new versions is
  89          done with correct sorting */
  90       string Version = List.Version();
  91       if (Version.empty() == true)
  92       {
  93          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
  94             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
  95          continue;
  96       }
  97
  98       pkgCache::VerIterator Ver = Pkg.VersionList();
  99       __apt_ptrloc *Last = &Pkg->VersionList;
 100       int Res = 1;
 101       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 102       {
 103          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 104                                  Ver.VerStr() + strlen(Ver.VerStr()));
 105          if (Res >= 0)
 106             break;
 107       }
 108
 109       /* We already have a version for this item, record that we
 110          saw it */
 111       if (Res == 0)
 112       {
 113          if (List.UsePackage(Pkg,Ver) == false)
 114             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 115
 116          if (NewFileVer(Ver,List) == false)
 117             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 118
 119          continue;
 120       }
 121
 122       // Add a new version
 123       *Last = NewVersion(Ver,Version,*Last);
 124       Ver->ParentPkg = Pkg.Index();
 125       if (List.NewVersion(Ver) == false)
 126          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 127
 128       if (List.UsePackage(Pkg,Ver) == false)
 129          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 130
 131       if (NewFileVer(Ver,List) == false)
 132          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 133    }
 134
 135    return true;
 136 }
 137                                                                         /*}}}*/
 138 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 139 // ---------------------------------------------------------------------
 140 /* This creates a new package structure and adds it to the hash table */
 141 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 142 {
 143    Pkg = Cache.FindPkg(Name);
 144    if (Pkg.end() == false)
 145       return true;
 146
 147    // Get a structure
 148    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 149    if (Package == 0)
 150       return false;
 151
 152    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 153
 154    // Insert it into the hash table
 155    unsigned long Hash = Cache.Hash(Name);
 156    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 157    Cache.HeaderP->HashTable[Hash] = Package;
 158
 159    // Set the name and the ID
 160    Pkg->Name = Map.WriteString(Name);
 161    if (Pkg->Name == 0)
 162       return false;
 163    Pkg->ID = Cache.HeaderP->PackageCount++;
 164
 165    return true;
 166 }
 167                                                                         /*}}}*/
 168 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 169 // ---------------------------------------------------------------------
 170 /* */
 171 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 172                                    ListParser &List)
 173 {
 174    // Get a structure
 175    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 176    if (VerFile == 0)
 177       return 0;
 178
 179    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 180    VF->File = CurrentFile - Cache.PkgFileP;
 181
 182    // Link it to the end of the list
 183    __apt_ptrloc *Last = &Ver->FileList;
 184    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 185       Last = &V->NextFile;
 186    VF->NextFile = *Last;
 187    *Last = VF.Index();
 188
 189    VF->Offset = List.Offset();
 190    VF->Size = List.Size();
 191    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 192       Cache.HeaderP->MaxVerFileSize = VF->Size;
 193    Cache.HeaderP->VerFileCount++;
 194
 195    return true;
 196 }
 197                                                                         /*}}}*/
 198 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 199 // ---------------------------------------------------------------------
 200 /* This puts a version structure in the linked list */
 201 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 202                                             string VerStr,
 203                                             unsigned long Next)
 204 {
 205    // Get a structure
 206    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 207    if (Version == 0)
 208       return 0;
 209
 210    // Fill it in
 211    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 212    Ver->NextVer = Next;
 213    Ver->ID = Cache.HeaderP->VersionCount++;
 214    Ver->VerStr = Map.WriteString(VerStr);
 215    if (Ver->VerStr == 0)
 216       return 0;
 217
 218    return Version;
 219 }
 220                                                                         /*}}}*/
 221 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 222 // ---------------------------------------------------------------------
 223 /* This creates a dependency element in the tree. It is linked to the
 224    version and to the package that it is pointing to. */
 225 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 226                                                string PackageName,
 227                                                string Version,
 228                                                unsigned int Op,
 229                                                unsigned int Type)
 230 {
 231    pkgCache &Cache = Owner->Cache;
 232
 233    // Get a structure
 234    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 235    if (Dependency == 0)
 236       return false;
 237
 238    // Fill it in
 239    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 240    Dep->ParentVer = Ver.Index();
 241    Dep->Type = Type;
 242    Dep->CompareOp = Op;
 243    Dep->ID = Cache.HeaderP->DependsCount++;
 244
 245    // Locate the target package
 246    pkgCache::PkgIterator Pkg;
 247    if (Owner->NewPackage(Pkg,PackageName) == false)
 248       return false;
 249
 250    // Probe the reverse dependency list for a version string that matches
 251    if (Version.empty() == false)
 252    {
 253       for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 254          if (I->Version != 0 && I.TargetVer() == Version)
 255             Dep->Version = I->Version;
 256       if (Dep->Version == 0)
 257          if ((Dep->Version = WriteString(Version)) == 0)
 258             return false;
 259    }
 260
 261    // Link it to the package
 262    Dep->Package = Pkg.Index();
 263    Dep->NextRevDepends = Pkg->RevDepends;
 264    Pkg->RevDepends = Dep.Index();
 265
 266    /* Link it to the version (at the end of the list)
 267       Caching the old end point speeds up generation substantially */
 268    static pkgCache::VerIterator OldVer(Cache);
 269    static __apt_ptrloc *OldLast;
 270    if (OldVer != Ver)
 271    {
 272       OldLast = &Ver->DependsList;
 273       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 274          OldLast = &D->NextDepends;
 275       OldVer = Ver;
 276    }
 277
 278    Dep->NextDepends = *OldLast;
 279    *OldLast = Dep.Index();
 280    OldLast = &Dep->NextDepends;
 281
 282    return true;
 283 }
 284                                                                         /*}}}*/
 285 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 286 // ---------------------------------------------------------------------
 287 /* */
 288 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 289                                                 string PackageName,
 290                                                 string Version)
 291 {
 292    pkgCache &Cache = Owner->Cache;
 293
 294    // We do not add self referencing provides
 295    if (Ver.ParentPkg().Name() == PackageName)
 296       return true;
 297
 298    // Get a structure
 299    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 300    if (Provides == 0)
 301       return false;
 302    Cache.HeaderP->ProvidesCount++;
 303
 304    // Fill it in
 305    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 306    Prv->Version = Ver.Index();
 307    Prv->NextPkgProv = Ver->ProvidesList;
 308    Ver->ProvidesList = Prv.Index();
 309    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 310       return false;
 311
 312    // Locate the target package
 313    pkgCache::PkgIterator Pkg;
 314    if (Owner->NewPackage(Pkg,PackageName) == false)
 315       return false;
 316
 317    // Link it to the package
 318    Prv->ParentPkg = Pkg.Index();
 319    Prv->NextProvides = Pkg->ProvidesList;
 320    Pkg->ProvidesList = Prv.Index();
 321
 322    return true;
 323 }
 324                                                                         /*}}}*/
 325 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 326 // ---------------------------------------------------------------------
 327 /* This is used to select which file is to be associated with all newly
 328    added versions. */
 329 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 330 {
 331    struct stat Buf;
 332    if (stat(File.c_str(),&Buf) == -1)
 333       return _error->Errno("stat","Couldn't stat ",File.c_str());
 334
 335    // Get some space for the structure
 336    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 337    if (CurrentFile == Cache.PkgFileP)
 338       return false;
 339
 340    // Fill it in
 341    CurrentFile->FileName = Map.WriteString(File);
 342    CurrentFile->Size = Buf.st_size;
 343    CurrentFile->mtime = Buf.st_mtime;
 344    CurrentFile->NextFile = Cache.HeaderP->FileList;
 345    CurrentFile->Flags = Flags;
 346    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 347    PkgFileName = File;
 348    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 349    Cache.HeaderP->PackageFileCount++;
 350
 351    if (CurrentFile->FileName == 0)
 352       return false;
 353
 354    Progress.SubProgress(Buf.st_size);
 355    return true;
 356 }
 357                                                                         /*}}}*/
 358 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 359 // ---------------------------------------------------------------------
 360 /* This is used to create handles to strings. Given the same text it
 361    always returns the same number */
 362 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 363                                                  unsigned int Size)
 364 {
 365    // Search for an insertion point
 366    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 367    int Res = 1;
 368    __apt_ptrloc *Last = &Cache.HeaderP->StringList;
 369    for (; I != Cache.StringItemP; Last = &I->NextItem,
 370         I = Cache.StringItemP + I->NextItem)
 371    {
 372       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 373       if (Res >= 0)
 374          break;
 375    }
 376
 377    // Match
 378    if (Res == 0)
 379       return I->String;
 380
 381    // Get a structure
 382    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 383    if (Item == 0)
 384       return 0;
 385
 386    // Fill in the structure
 387    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 388    ItemP->NextItem = I - Cache.StringItemP;
 389    *Last = Item;
 390    ItemP->String = Map.WriteString(S,Size);
 391    if (ItemP->String == 0)
 392       return 0;
 393
 394    return ItemP->String;
 395 }
 396                                                                         /*}}}*/
 397
 398 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 399 // ---------------------------------------------------------------------
 400 /* The source cache is checked against the source list and the files
 401    on disk, any difference results in a false. */
 402 bool pkgSrcCacheCheck(pkgSourceList &List)
 403 {
 404    if (_error->PendingError() == true)
 405       return false;
 406
 407    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 408    string ListDir = _config->FindDir("Dir::State::lists");
 409
 410    // Count the number of missing files
 411    int Missing = 0;
 412    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 413    {
 414       string File = ListDir + URItoFileName(I->PackagesURI());
 415       struct stat Buf;
 416       if (stat(File.c_str(),&Buf) != 0)
 417       {
 418          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 419                           I->PackagesInfo().c_str(),File.c_str());
 420          Missing++;
 421       }
 422    }
 423
 424    // Open the source package cache
 425    if (FileExists(CacheFile) == false)
 426       return false;
 427
 428    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 429    if (_error->PendingError() == true)
 430    {
 431       _error->Discard();
 432       return false;
 433    }
 434
 435    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 436    if (_error->PendingError() == true || Map.Size() == 0)
 437    {
 438       _error->Discard();
 439       return false;
 440    }
 441
 442    pkgCache Cache(Map);
 443    if (_error->PendingError() == true)
 444    {
 445       _error->Discard();
 446       return false;
 447    }
 448
 449    // They are certianly out of sync
 450    if (Cache.Head().PackageFileCount != List.size() - Missing)
 451       return false;
 452
 453    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 454    {
 455       // Search for a match in the source list
 456       bool Bad = true;
 457       for (pkgSourceList::const_iterator I = List.begin();
 458            I != List.end(); I++)
 459       {
 460          string File = ListDir + URItoFileName(I->PackagesURI());
 461          if (F.FileName() == File)
 462          {
 463             Bad = false;
 464             break;
 465          }
 466       }
 467
 468       // Check if the file matches what was cached
 469       Bad |= !F.IsOk();
 470       if (Bad == true)
 471          return false;
 472    }
 473
 474    return true;
 475 }
 476                                                                         /*}}}*/
 477 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 478 // ---------------------------------------------------------------------
 479 /* This does a simple check of all files used to compose the cache */
 480 bool pkgPkgCacheCheck(string CacheFile)
 481 {
 482    if (_error->PendingError() == true)
 483       return false;
 484
 485    // Open the source package cache
 486    if (FileExists(CacheFile) == false)
 487       return false;
 488
 489    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 490    if (_error->PendingError() == true)
 491    {
 492       _error->Discard();
 493       return false;
 494    }
 495
 496    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 497    if (_error->PendingError() == true || Map.Size() == 0)
 498    {
 499       _error->Discard();
 500       return false;
 501    }
 502
 503    pkgCache Cache(Map);
 504    if (_error->PendingError() == true)
 505    {
 506       _error->Discard();
 507       return false;
 508    }
 509
 510    // Status files that must be in the cache
 511    string Status[3];
 512    Status[0] = _config->FindFile("Dir::State::xstatus");
 513    Status[1]= _config->FindFile("Dir::State::userstatus");
 514    Status[2] = _config->FindFile("Dir::State::status");
 515
 516    // Cheack each file
 517    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 518    {
 519       if (F.IsOk() == false)
 520          return false;
 521
 522       // See if this is one of the status files
 523       for (int I = 0; I != 3; I++)
 524          if (F.FileName() == Status[I])
 525             Status[I] = string();
 526    }
 527
 528    // Make sure all the status files are loaded.
 529    for (int I = 0; I != 3; I++)
 530    {
 531       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 532          return false;
 533    }
 534
 535    return true;
 536 }
 537                                                                         /*}}}*/
 538 // AddSourcesSize - Add the size of the status files                    /*{{{*/
 539 // ---------------------------------------------------------------------
 540 /* This adds the size of all the status files to the size counter */
 541 static bool pkgAddSourcesSize(unsigned long &TotalSize)
 542 {
 543    // Grab the file names
 544    string xstatus = _config->FindFile("Dir::State::xstatus");
 545    string userstatus = _config->FindFile("Dir::State::userstatus");
 546    string status = _config->FindFile("Dir::State::status");
 547
 548    // Grab the sizes
 549    struct stat Buf;
 550    if (stat(xstatus.c_str(),&Buf) == 0)
 551       TotalSize += Buf.st_size;
 552    if (stat(userstatus.c_str(),&Buf) == 0)
 553       TotalSize += Buf.st_size;
 554    if (stat(status.c_str(),&Buf) != 0)
 555       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 556    TotalSize += Buf.st_size;
 557
 558    return true;
 559 }
 560                                                                         /*}}}*/
 561 // MergeStatus - Add the status files to the cache                      /*{{{*/
 562 // ---------------------------------------------------------------------
 563 /* This adds the status files to the map */
 564 static bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 565                            unsigned long &CurrentSize,unsigned long TotalSize)
 566 {
 567    // Grab the file names
 568    string Status[3];
 569    Status[0] = _config->FindFile("Dir::State::xstatus");
 570    Status[1]= _config->FindFile("Dir::State::userstatus");
 571    Status[2] = _config->FindFile("Dir::State::status");
 572
 573    for (int I = 0; I != 3; I++)
 574    {
 575       // Check if the file exists and it is not the primary status file.
 576       string File = Status[I];
 577       if (I != 2 && FileExists(File) == false)
 578          continue;
 579
 580       FileFd Pkg(File,FileFd::ReadOnly);
 581       debListParser Parser(Pkg);
 582       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 583       if (_error->PendingError() == true)
 584          return _error->Error("Problem opening %s",File.c_str());
 585       CurrentSize += Pkg.Size();
 586
 587       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 588       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 589          return _error->Error("Problem with SelectFile %s",File.c_str());
 590
 591       if (Gen.MergeList(Parser) == false)
 592          return _error->Error("Problem with MergeList %s",File.c_str());
 593       Progress.Progress(Pkg.Size());
 594    }
 595
 596    return true;
 597 }
 598                                                                         /*}}}*/
 599 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* This copies the package source cache and then merges the status and
 602    xstatus files into it. */
 603 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 604 {
 605    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 606
 607    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 608    bool SrcOk = pkgSrcCacheCheck(List);
 609    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 610
 611    // Rebuild the source and package caches
 612    if (SrcOk == false)
 613    {
 614       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 615       string ListDir = _config->FindDir("Dir::State::lists");
 616       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 617       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 618       DynamicMMap Map(CacheF,MMap::Public);
 619       if (_error->PendingError() == true)
 620          return false;
 621
 622       pkgCacheGenerator Gen(Map,Progress);
 623
 624       // Prepare the progress indicator
 625       unsigned long TotalSize = 0;
 626       struct stat Buf;
 627       for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 628       {
 629          string File = ListDir + URItoFileName(I->PackagesURI());
 630          if (stat(File.c_str(),&Buf) != 0)
 631             continue;
 632          TotalSize += Buf.st_size;
 633       }
 634
 635       if (pkgAddSourcesSize(TotalSize) == false)
 636          return false;
 637
 638       // Generate the pkg source cache
 639       unsigned long CurrentSize = 0;
 640       for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 641       {
 642          string File = ListDir + URItoFileName(I->PackagesURI());
 643
 644          if (FileExists(File) == false)
 645             continue;
 646
 647          FileFd Pkg(File,FileFd::ReadOnly);
 648          debListParser Parser(Pkg);
 649          Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 650          if (_error->PendingError() == true)
 651             return _error->Error("Problem opening %s",File.c_str());
 652          CurrentSize += Pkg.Size();
 653
 654          Progress.SubProgress(0,I->PackagesInfo());
 655          if (Gen.SelectFile(File) == false)
 656             return _error->Error("Problem with SelectFile %s",File.c_str());
 657
 658          if (Gen.MergeList(Parser) == false)
 659             return _error->Error("Problem with MergeList %s",File.c_str());
 660
 661          // Check the release file
 662          string RFile = ListDir + URItoFileName(I->ReleaseURI());
 663          if (FileExists(RFile) == true)
 664          {
 665             FileFd Rel(RFile,FileFd::ReadOnly);
 666             if (_error->PendingError() == true)
 667                return false;
 668             Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 669          }
 670       }
 671
 672       // Write the src cache
 673       Gen.GetCache().HeaderP->Dirty = false;
 674       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 675          return _error->Error("IO Error saving source cache");
 676       Gen.GetCache().HeaderP->Dirty = true;
 677
 678       // Merge in the source caches
 679       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 680    }
 681
 682    if (PkgOk == true)
 683    {
 684       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 685       return true;
 686    }
 687
 688    // We use the source cache to generate the package cache
 689    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 690
 691    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 692    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 693    DynamicMMap Map(CacheF,MMap::Public);
 694    if (_error->PendingError() == true)
 695       return false;
 696
 697    // Preload the map with the source cache
 698    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 699                     SCacheF.Size()) == false)
 700       return false;
 701
 702    pkgCacheGenerator Gen(Map,Progress);
 703
 704    // Compute the progress
 705    unsigned long TotalSize = 0;
 706    if (pkgAddSourcesSize(TotalSize) == false)
 707       return false;
 708
 709    unsigned long CurrentSize = 0;
 710    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 711 }
 712                                                                         /*}}}*/