apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.26 1998/12/14 03:39:15 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24
  25 #include <strutl.h>
  26
  27 #include <sys/stat.h>
  28 #include <unistd.h>
  29                                                                         /*}}}*/
  30
  31 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  32 // ---------------------------------------------------------------------
  33 /* We set the diry flag and make sure that is written to the disk */
  34 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  35                     Map(Map), Cache(Map), Progress(Prog)
  36 {
  37    if (_error->PendingError() == true)
  38       return;
  39
  40    if (Map.Size() == 0)
  41    {
  42       Map.RawAllocate(sizeof(pkgCache::Header));
  43       *Cache.HeaderP = pkgCache::Header();
  44    }
  45    Cache.HeaderP->Dirty = true;
  46    Map.Sync(0,sizeof(pkgCache::Header));
  47    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  48 }
  49                                                                         /*}}}*/
  50 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* We sync the data then unset the dirty flag in two steps so as to
  53    advoid a problem during a crash */
  54 pkgCacheGenerator::~pkgCacheGenerator()
  55 {
  56    if (_error->PendingError() == true)
  57       return;
  58    if (Map.Sync() == false)
  59       return;
  60
  61    Cache.HeaderP->Dirty = false;
  62    Map.Sync(0,sizeof(pkgCache::Header));
  63 }
  64                                                                         /*}}}*/
  65 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* This provides the generation of the entries in the cache. Each loop
  68    goes through a single package record from the underlying parse engine. */
  69 bool pkgCacheGenerator::MergeList(ListParser &List)
  70 {
  71    List.Owner = this;
  72
  73    int Counter = 0;
  74    while (List.Step() == true)
  75    {
  76       // Get a pointer to the package structure
  77       string PackageName = List.Package();
  78       pkgCache::PkgIterator Pkg;
  79       if (NewPackage(Pkg,PackageName) == false)
  80          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  81       Counter++;
  82       if (Counter % 100 == 0)
  83          Progress.Progress(List.Offset());
  84
  85       /* Get a pointer to the version structure. We know the list is sorted
  86          so we use that fact in the search. Insertion of new versions is
  87          done with correct sorting */
  88       string Version = List.Version();
  89       if (Version.empty() == true)
  90       {
  91          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
  92             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
  93          continue;
  94       }
  95
  96       pkgCache::VerIterator Ver = Pkg.VersionList();
  97       __apt_ptrloc *Last = &Pkg->VersionList;
  98       int Res = 1;
  99       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 100       {
 101          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 102                                  Ver.VerStr() + strlen(Ver.VerStr()));
 103          if (Res >= 0)
 104             break;
 105       }
 106
 107       /* We already have a version for this item, record that we
 108          saw it */
 109       if (Res == 0)
 110       {
 111          if (List.UsePackage(Pkg,Ver) == false)
 112             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 113
 114          if (NewFileVer(Ver,List) == false)
 115             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 116
 117          continue;
 118       }
 119
 120       // Add a new version
 121       *Last = NewVersion(Ver,Version,*Last);
 122       Ver->ParentPkg = Pkg.Index();
 123       if (List.NewVersion(Ver) == false)
 124          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 125
 126       if (List.UsePackage(Pkg,Ver) == false)
 127          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 128
 129       if (NewFileVer(Ver,List) == false)
 130          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 131    }
 132
 133    return true;
 134 }
 135                                                                         /*}}}*/
 136 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 137 // ---------------------------------------------------------------------
 138 /* This creates a new package structure and adds it to the hash table */
 139 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 140 {
 141    Pkg = Cache.FindPkg(Name);
 142    if (Pkg.end() == false)
 143       return true;
 144
 145    // Get a structure
 146    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 147    if (Package == 0)
 148       return false;
 149
 150    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 151
 152    // Insert it into the hash table
 153    unsigned long Hash = Cache.Hash(Name);
 154    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 155    Cache.HeaderP->HashTable[Hash] = Package;
 156
 157    // Set the name and the ID
 158    Pkg->Name = Map.WriteString(Name);
 159    if (Pkg->Name == 0)
 160       return false;
 161    Pkg->ID = Cache.HeaderP->PackageCount++;
 162
 163    return true;
 164 }
 165                                                                         /*}}}*/
 166 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 167 // ---------------------------------------------------------------------
 168 /* */
 169 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 170                                    ListParser &List)
 171 {
 172    // Get a structure
 173    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 174    if (VerFile == 0)
 175       return 0;
 176
 177    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 178    VF->File = CurrentFile - Cache.PkgFileP;
 179
 180    // Link it to the end of the list
 181    __apt_ptrloc *Last = &Ver->FileList;
 182    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 183       Last = &V->NextFile;
 184    VF->NextFile = *Last;
 185    *Last = VF.Index();
 186
 187    VF->Offset = List.Offset();
 188    VF->Size = List.Size();
 189    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 190       Cache.HeaderP->MaxVerFileSize = VF->Size;
 191    Cache.HeaderP->VerFileCount++;
 192
 193    return true;
 194 }
 195                                                                         /*}}}*/
 196 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* This puts a version structure in the linked list */
 199 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 200                                             string VerStr,
 201                                             unsigned long Next)
 202 {
 203    // Get a structure
 204    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 205    if (Version == 0)
 206       return 0;
 207
 208    // Fill it in
 209    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 210    Ver->NextVer = Next;
 211    Ver->ID = Cache.HeaderP->VersionCount++;
 212    Ver->VerStr = Map.WriteString(VerStr);
 213    if (Ver->VerStr == 0)
 214       return 0;
 215
 216    return Version;
 217 }
 218                                                                         /*}}}*/
 219 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 220 // ---------------------------------------------------------------------
 221 /* This creates a dependency element in the tree. It is linked to the
 222    version and to the package that it is pointing to. */
 223 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 224                                                string PackageName,
 225                                                string Version,
 226                                                unsigned int Op,
 227                                                unsigned int Type)
 228 {
 229    pkgCache &Cache = Owner->Cache;
 230
 231    // Get a structure
 232    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 233    if (Dependency == 0)
 234       return false;
 235
 236    // Fill it in
 237    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 238    Dep->ParentVer = Ver.Index();
 239    Dep->Type = Type;
 240    Dep->CompareOp = Op;
 241    Dep->ID = Cache.HeaderP->DependsCount++;
 242
 243    // Locate the target package
 244    pkgCache::PkgIterator Pkg;
 245    if (Owner->NewPackage(Pkg,PackageName) == false)
 246       return false;
 247
 248    // Probe the reverse dependency list for a version string that matches
 249    if (Version.empty() == false)
 250    {
 251       for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 252          if (I->Version != 0 && I.TargetVer() == Version)
 253             Dep->Version = I->Version;
 254       if (Dep->Version == 0)
 255          if ((Dep->Version = WriteString(Version)) == 0)
 256             return false;
 257    }
 258
 259    // Link it to the package
 260    Dep->Package = Pkg.Index();
 261    Dep->NextRevDepends = Pkg->RevDepends;
 262    Pkg->RevDepends = Dep.Index();
 263
 264    /* Link it to the version (at the end of the list)
 265       Caching the old end point speeds up generation substantially */
 266    static pkgCache::VerIterator OldVer(Cache);
 267    static __apt_ptrloc *OldLast;
 268    if (OldVer != Ver)
 269    {
 270       OldLast = &Ver->DependsList;
 271       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 272          OldLast = &D->NextDepends;
 273       OldVer = Ver;
 274    }
 275
 276    Dep->NextDepends = *OldLast;
 277    *OldLast = Dep.Index();
 278    OldLast = &Dep->NextDepends;
 279
 280    return true;
 281 }
 282                                                                         /*}}}*/
 283 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 284 // ---------------------------------------------------------------------
 285 /* */
 286 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 287                                                 string PackageName,
 288                                                 string Version)
 289 {
 290    pkgCache &Cache = Owner->Cache;
 291
 292    // We do not add self referencing provides
 293    if (Ver.ParentPkg().Name() == PackageName)
 294       return true;
 295
 296    // Get a structure
 297    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 298    if (Provides == 0)
 299       return false;
 300    Cache.HeaderP->ProvidesCount++;
 301
 302    // Fill it in
 303    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 304    Prv->Version = Ver.Index();
 305    Prv->NextPkgProv = Ver->ProvidesList;
 306    Ver->ProvidesList = Prv.Index();
 307    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 308       return false;
 309
 310    // Locate the target package
 311    pkgCache::PkgIterator Pkg;
 312    if (Owner->NewPackage(Pkg,PackageName) == false)
 313       return false;
 314
 315    // Link it to the package
 316    Prv->ParentPkg = Pkg.Index();
 317    Prv->NextProvides = Pkg->ProvidesList;
 318    Pkg->ProvidesList = Prv.Index();
 319
 320    return true;
 321 }
 322                                                                         /*}}}*/
 323 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 324 // ---------------------------------------------------------------------
 325 /* This is used to select which file is to be associated with all newly
 326    added versions. */
 327 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 328 {
 329    struct stat Buf;
 330    if (stat(File.c_str(),&Buf) == -1)
 331       return _error->Errno("stat","Couldn't stat ",File.c_str());
 332
 333    // Get some space for the structure
 334    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 335    if (CurrentFile == Cache.PkgFileP)
 336       return false;
 337
 338    // Fill it in
 339    CurrentFile->FileName = Map.WriteString(File);
 340    CurrentFile->Size = Buf.st_size;
 341    CurrentFile->mtime = Buf.st_mtime;
 342    CurrentFile->NextFile = Cache.HeaderP->FileList;
 343    CurrentFile->Flags = Flags;
 344    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 345    PkgFileName = File;
 346    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 347    Cache.HeaderP->PackageFileCount++;
 348
 349    if (CurrentFile->FileName == 0)
 350       return false;
 351
 352    Progress.SubProgress(Buf.st_size);
 353    return true;
 354 }
 355                                                                         /*}}}*/
 356 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 357 // ---------------------------------------------------------------------
 358 /* This is used to create handles to strings. Given the same text it
 359    always returns the same number */
 360 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 361                                                  unsigned int Size)
 362 {
 363    // Search for an insertion point
 364    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 365    int Res = 1;
 366    __apt_ptrloc *Last = &Cache.HeaderP->StringList;
 367    for (; I != Cache.StringItemP; Last = &I->NextItem,
 368         I = Cache.StringItemP + I->NextItem)
 369    {
 370       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 371       if (Res >= 0)
 372          break;
 373    }
 374
 375    // Match
 376    if (Res == 0)
 377       return I->String;
 378
 379    // Get a structure
 380    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 381    if (Item == 0)
 382       return 0;
 383
 384    // Fill in the structure
 385    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 386    ItemP->NextItem = I - Cache.StringItemP;
 387    *Last = Item;
 388    ItemP->String = Map.WriteString(S,Size);
 389    if (ItemP->String == 0)
 390       return 0;
 391
 392    return ItemP->String;
 393 }
 394                                                                         /*}}}*/
 395
 396 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 397 // ---------------------------------------------------------------------
 398 /* The source cache is checked against the source list and the files
 399    on disk, any difference results in a false. */
 400 bool pkgSrcCacheCheck(pkgSourceList &List)
 401 {
 402    if (_error->PendingError() == true)
 403       return false;
 404
 405    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 406    string ListDir = _config->FindDir("Dir::State::lists");
 407
 408    // Count the number of missing files
 409    int Missing = 0;
 410    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 411    {
 412       string File = ListDir + URItoFileName(I->PackagesURI());
 413       struct stat Buf;
 414       if (stat(File.c_str(),&Buf) != 0)
 415       {
 416          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 417                           I->PackagesInfo().c_str(),File.c_str());
 418          Missing++;
 419       }
 420    }
 421
 422    // Open the source package cache
 423    if (FileExists(CacheFile) == false)
 424       return false;
 425
 426    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 427    if (_error->PendingError() == true)
 428    {
 429       _error->Discard();
 430       return false;
 431    }
 432
 433    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 434    if (_error->PendingError() == true || Map.Size() == 0)
 435    {
 436       _error->Discard();
 437       return false;
 438    }
 439
 440    pkgCache Cache(Map);
 441    if (_error->PendingError() == true)
 442    {
 443       _error->Discard();
 444       return false;
 445    }
 446
 447    // They are certianly out of sync
 448    if (Cache.Head().PackageFileCount != List.size() - Missing)
 449       return false;
 450
 451    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 452    {
 453       // Search for a match in the source list
 454       bool Bad = true;
 455       for (pkgSourceList::const_iterator I = List.begin();
 456            I != List.end(); I++)
 457       {
 458          string File = ListDir + URItoFileName(I->PackagesURI());
 459          if (F.FileName() == File)
 460          {
 461             Bad = false;
 462             break;
 463          }
 464       }
 465
 466       // Check if the file matches what was cached
 467       Bad |= !F.IsOk();
 468       if (Bad == true)
 469          return false;
 470    }
 471
 472    return true;
 473 }
 474                                                                         /*}}}*/
 475 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 476 // ---------------------------------------------------------------------
 477 /* This does a simple check of all files used to compose the cache */
 478 bool pkgPkgCacheCheck(string CacheFile)
 479 {
 480    if (_error->PendingError() == true)
 481       return false;
 482
 483    // Open the source package cache
 484    if (FileExists(CacheFile) == false)
 485       return false;
 486
 487    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 488    if (_error->PendingError() == true)
 489    {
 490       _error->Discard();
 491       return false;
 492    }
 493
 494    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 495    if (_error->PendingError() == true || Map.Size() == 0)
 496    {
 497       _error->Discard();
 498       return false;
 499    }
 500
 501    pkgCache Cache(Map);
 502    if (_error->PendingError() == true)
 503    {
 504       _error->Discard();
 505       return false;
 506    }
 507
 508    // Status files that must be in the cache
 509    string Status[3];
 510    Status[0] = _config->FindFile("Dir::State::xstatus");
 511    Status[1]= _config->FindFile("Dir::State::userstatus");
 512    Status[2] = _config->FindFile("Dir::State::status");
 513
 514    // Cheack each file
 515    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 516    {
 517       if (F.IsOk() == false)
 518          return false;
 519
 520       // See if this is one of the status files
 521       for (int I = 0; I != 3; I++)
 522          if (F.FileName() == Status[I])
 523             Status[I] = string();
 524    }
 525
 526    // Make sure all the status files are loaded.
 527    for (int I = 0; I != 3; I++)
 528    {
 529       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 530          return false;
 531    }
 532
 533    return true;
 534 }
 535                                                                         /*}}}*/
 536 // AddSourcesSize - Add the size of the status files                    /*{{{*/
 537 // ---------------------------------------------------------------------
 538 /* This adds the size of all the status files to the size counter */
 539 static bool pkgAddSourcesSize(unsigned long &TotalSize)
 540 {
 541    // Grab the file names
 542    string xstatus = _config->FindFile("Dir::State::xstatus");
 543    string userstatus = _config->FindFile("Dir::State::userstatus");
 544    string status = _config->FindFile("Dir::State::status");
 545
 546    // Grab the sizes
 547    struct stat Buf;
 548    if (stat(xstatus.c_str(),&Buf) == 0)
 549       TotalSize += Buf.st_size;
 550    if (stat(userstatus.c_str(),&Buf) == 0)
 551       TotalSize += Buf.st_size;
 552    if (stat(status.c_str(),&Buf) != 0)
 553       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 554    TotalSize += Buf.st_size;
 555
 556    return true;
 557 }
 558                                                                         /*}}}*/
 559 // MergeStatus - Add the status files to the cache                      /*{{{*/
 560 // ---------------------------------------------------------------------
 561 /* This adds the status files to the map */
 562 static bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 563                            unsigned long &CurrentSize,unsigned long TotalSize)
 564 {
 565    // Grab the file names
 566    string Status[3];
 567    Status[0] = _config->FindFile("Dir::State::xstatus");
 568    Status[1]= _config->FindFile("Dir::State::userstatus");
 569    Status[2] = _config->FindFile("Dir::State::status");
 570
 571    for (int I = 0; I != 3; I++)
 572    {
 573       // Check if the file exists and it is not the primary status file.
 574       string File = Status[I];
 575       if (I != 2 && FileExists(File) == false)
 576          continue;
 577
 578       FileFd Pkg(File,FileFd::ReadOnly);
 579       debListParser Parser(Pkg);
 580       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 581       if (_error->PendingError() == true)
 582          return _error->Error("Problem opening %s",File.c_str());
 583       CurrentSize += Pkg.Size();
 584
 585       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 586       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 587          return _error->Error("Problem with SelectFile %s",File.c_str());
 588
 589       if (Gen.MergeList(Parser) == false)
 590          return _error->Error("Problem with MergeList %s",File.c_str());
 591       Progress.Progress(Pkg.Size());
 592    }
 593
 594    return true;
 595 }
 596                                                                         /*}}}*/
 597 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* This copies the package source cache and then merges the status and
 600    xstatus files into it. */
 601 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 602 {
 603    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 604
 605    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 606    bool SrcOk = pkgSrcCacheCheck(List);
 607    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 608
 609    // Rebuild the source and package caches
 610    if (SrcOk == false)
 611    {
 612       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 613       string ListDir = _config->FindDir("Dir::State::lists");
 614       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 615       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 616       DynamicMMap Map(CacheF,MMap::Public);
 617       if (_error->PendingError() == true)
 618          return false;
 619
 620       pkgCacheGenerator Gen(Map,Progress);
 621
 622       // Prepare the progress indicator
 623       unsigned long TotalSize = 0;
 624       struct stat Buf;
 625       for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 626       {
 627          string File = ListDir + URItoFileName(I->PackagesURI());
 628          if (stat(File.c_str(),&Buf) != 0)
 629             continue;
 630          TotalSize += Buf.st_size;
 631       }
 632
 633       if (pkgAddSourcesSize(TotalSize) == false)
 634          return false;
 635
 636       // Generate the pkg source cache
 637       unsigned long CurrentSize = 0;
 638       for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 639       {
 640          string File = ListDir + URItoFileName(I->PackagesURI());
 641
 642          if (FileExists(File) == false)
 643             continue;
 644
 645          FileFd Pkg(File,FileFd::ReadOnly);
 646          debListParser Parser(Pkg);
 647          Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 648          if (_error->PendingError() == true)
 649             return _error->Error("Problem opening %s",File.c_str());
 650          CurrentSize += Pkg.Size();
 651
 652          Progress.SubProgress(0,I->PackagesInfo());
 653          if (Gen.SelectFile(File) == false)
 654             return _error->Error("Problem with SelectFile %s",File.c_str());
 655
 656          if (Gen.MergeList(Parser) == false)
 657             return _error->Error("Problem with MergeList %s",File.c_str());
 658
 659          // Check the release file
 660          string RFile = ListDir + URItoFileName(I->ReleaseURI());
 661          if (FileExists(RFile) == true)
 662          {
 663             FileFd Rel(RFile,FileFd::ReadOnly);
 664             if (_error->PendingError() == true)
 665                return false;
 666             Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 667          }
 668       }
 669
 670       // Write the src cache
 671       Gen.GetCache().HeaderP->Dirty = false;
 672       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 673          return _error->Error("IO Error saving source cache");
 674       Gen.GetCache().HeaderP->Dirty = true;
 675
 676       // Merge in the source caches
 677       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 678    }
 679
 680    if (PkgOk == true)
 681    {
 682       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 683       return true;
 684    }
 685
 686    // We use the source cache to generate the package cache
 687    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 688
 689    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 690    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 691    DynamicMMap Map(CacheF,MMap::Public);
 692    if (_error->PendingError() == true)
 693       return false;
 694
 695    // Preload the map with the source cache
 696    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 697                     SCacheF.Size()) == false)
 698       return false;
 699
 700    pkgCacheGenerator Gen(Map,Progress);
 701
 702    // Compute the progress
 703    unsigned long TotalSize = 0;
 704    if (pkgAddSourcesSize(TotalSize) == false)
 705       return false;
 706
 707    unsigned long CurrentSize = 0;
 708    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 709 }
 710                                                                         /*}}}*/