apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.43 1999/12/10 23:40:29 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25
  26 #include <sys/stat.h>
  27 #include <unistd.h>
  28 #include <errno.h>
  29
  30 #include <system.h>
  31                                                                         /*}}}*/
  32
  33 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  34 // ---------------------------------------------------------------------
  35 /* We set the diry flag and make sure that is written to the disk */
  36 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  37                     Map(Map), Cache(Map), Progress(&Prog)
  38 {
  39    CurrentFile = 0;
  40
  41    if (_error->PendingError() == true)
  42       return;
  43
  44    if (Map.Size() == 0)
  45    {
  46       Map.RawAllocate(sizeof(pkgCache::Header));
  47       *Cache.HeaderP = pkgCache::Header();
  48    }
  49    Cache.HeaderP->Dirty = true;
  50    Map.Sync(0,sizeof(pkgCache::Header));
  51    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  52    memset(UniqHash,0,sizeof(UniqHash));
  53 }
  54                                                                         /*}}}*/
  55 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* We sync the data then unset the dirty flag in two steps so as to
  58    advoid a problem during a crash */
  59 pkgCacheGenerator::~pkgCacheGenerator()
  60 {
  61    if (_error->PendingError() == true)
  62       return;
  63    if (Map.Sync() == false)
  64       return;
  65
  66    Cache.HeaderP->Dirty = false;
  67    Map.Sync(0,sizeof(pkgCache::Header));
  68 }
  69                                                                         /*}}}*/
  70 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  71 // ---------------------------------------------------------------------
  72 /* This provides the generation of the entries in the cache. Each loop
  73    goes through a single package record from the underlying parse engine. */
  74 bool pkgCacheGenerator::MergeList(ListParser &List,
  75                                   pkgCache::VerIterator *OutVer)
  76 {
  77    List.Owner = this;
  78
  79    unsigned int Counter = 0;
  80    while (List.Step() == true)
  81    {
  82       // Get a pointer to the package structure
  83       string PackageName = List.Package();
  84       if (PackageName.empty() == true)
  85          return false;
  86
  87       pkgCache::PkgIterator Pkg;
  88       if (NewPackage(Pkg,PackageName) == false)
  89          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  90       Counter++;
  91       if (Counter % 100 == 0 && Progress != 0)
  92          Progress->Progress(List.Offset());
  93
  94       /* Get a pointer to the version structure. We know the list is sorted
  95          so we use that fact in the search. Insertion of new versions is
  96          done with correct sorting */
  97       string Version = List.Version();
  98       if (Version.empty() == true)
  99       {
 100          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 101             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
 102          continue;
 103       }
 104
 105       pkgCache::VerIterator Ver = Pkg.VersionList();
 106       map_ptrloc *Last = &Pkg->VersionList;
 107       int Res = 1;
 108       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 109       {
 110          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 111                                  Ver.VerStr() + strlen(Ver.VerStr()));
 112          if (Res >= 0)
 113             break;
 114       }
 115
 116       /* We already have a version for this item, record that we
 117          saw it */
 118       unsigned long Hash = List.VersionHash();
 119       if (Res == 0 && Ver->Hash == Hash)
 120       {
 121          if (List.UsePackage(Pkg,Ver) == false)
 122             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 123
 124          if (NewFileVer(Ver,List) == false)
 125             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 126
 127          // Read only a single record and return
 128          if (OutVer != 0)
 129          {
 130             *OutVer = Ver;
 131             return true;
 132          }
 133
 134          continue;
 135       }
 136
 137       // Skip to the end of the same version set.
 138       if (Res == 0)
 139       {
 140          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 141          {
 142             Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 143                                     Ver.VerStr() + strlen(Ver.VerStr()));
 144             if (Res != 0)
 145                break;
 146          }
 147       }
 148
 149       // Add a new version
 150       *Last = NewVersion(Ver,Version,*Last);
 151       Ver->ParentPkg = Pkg.Index();
 152       Ver->Hash = Hash;
 153       if (List.NewVersion(Ver) == false)
 154          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 155
 156       if (List.UsePackage(Pkg,Ver) == false)
 157          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 158
 159       if (NewFileVer(Ver,List) == false)
 160          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 161
 162       // Read only a single record and return
 163       if (OutVer != 0)
 164       {
 165          *OutVer = Ver;
 166          return true;
 167       }
 168    }
 169
 170    return true;
 171 }
 172                                                                         /*}}}*/
 173 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 174 // ---------------------------------------------------------------------
 175 /* This creates a new package structure and adds it to the hash table */
 176 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 177 {
 178    Pkg = Cache.FindPkg(Name);
 179    if (Pkg.end() == false)
 180       return true;
 181
 182    // Get a structure
 183    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 184    if (Package == 0)
 185       return false;
 186
 187    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 188
 189    // Insert it into the hash table
 190    unsigned long Hash = Cache.Hash(Name);
 191    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 192    Cache.HeaderP->HashTable[Hash] = Package;
 193
 194    // Set the name and the ID
 195    Pkg->Name = Map.WriteString(Name);
 196    if (Pkg->Name == 0)
 197       return false;
 198    Pkg->ID = Cache.HeaderP->PackageCount++;
 199
 200    return true;
 201 }
 202                                                                         /*}}}*/
 203 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 204 // ---------------------------------------------------------------------
 205 /* */
 206 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 207                                    ListParser &List)
 208 {
 209    if (CurrentFile == 0)
 210       return true;
 211
 212    // Get a structure
 213    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 214    if (VerFile == 0)
 215       return 0;
 216
 217    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 218    VF->File = CurrentFile - Cache.PkgFileP;
 219
 220    // Link it to the end of the list
 221    map_ptrloc *Last = &Ver->FileList;
 222    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 223       Last = &V->NextFile;
 224    VF->NextFile = *Last;
 225    *Last = VF.Index();
 226
 227    VF->Offset = List.Offset();
 228    VF->Size = List.Size();
 229    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 230       Cache.HeaderP->MaxVerFileSize = VF->Size;
 231    Cache.HeaderP->VerFileCount++;
 232
 233    return true;
 234 }
 235                                                                         /*}}}*/
 236 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 237 // ---------------------------------------------------------------------
 238 /* This puts a version structure in the linked list */
 239 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 240                                             string VerStr,
 241                                             unsigned long Next)
 242 {
 243    // Get a structure
 244    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 245    if (Version == 0)
 246       return 0;
 247
 248    // Fill it in
 249    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 250    Ver->NextVer = Next;
 251    Ver->ID = Cache.HeaderP->VersionCount++;
 252    Ver->VerStr = Map.WriteString(VerStr);
 253    if (Ver->VerStr == 0)
 254       return 0;
 255
 256    return Version;
 257 }
 258                                                                         /*}}}*/
 259 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 260 // ---------------------------------------------------------------------
 261 /* This creates a dependency element in the tree. It is linked to the
 262    version and to the package that it is pointing to. */
 263 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 264                                                string PackageName,
 265                                                string Version,
 266                                                unsigned int Op,
 267                                                unsigned int Type)
 268 {
 269    pkgCache &Cache = Owner->Cache;
 270
 271    // Get a structure
 272    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 273    if (Dependency == 0)
 274       return false;
 275
 276    // Fill it in
 277    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 278    Dep->ParentVer = Ver.Index();
 279    Dep->Type = Type;
 280    Dep->CompareOp = Op;
 281    Dep->ID = Cache.HeaderP->DependsCount++;
 282
 283    // Locate the target package
 284    pkgCache::PkgIterator Pkg;
 285    if (Owner->NewPackage(Pkg,PackageName) == false)
 286       return false;
 287
 288    // Probe the reverse dependency list for a version string that matches
 289    if (Version.empty() == false)
 290    {
 291 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++, Hit++)
 292          if (I->Version != 0 && I.TargetVer() == Version)
 293             Dep->Version = I->Version;*/
 294       if (Dep->Version == 0)
 295          if ((Dep->Version = WriteString(Version)) == 0)
 296             return false;
 297    }
 298
 299    // Link it to the package
 300    Dep->Package = Pkg.Index();
 301    Dep->NextRevDepends = Pkg->RevDepends;
 302    Pkg->RevDepends = Dep.Index();
 303
 304    /* Link it to the version (at the end of the list)
 305       Caching the old end point speeds up generation substantially */
 306    if (OldDepVer != Ver)
 307    {
 308       OldDepLast = &Ver->DependsList;
 309       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 310          OldDepLast = &D->NextDepends;
 311       OldDepVer = Ver;
 312    }
 313
 314    Dep->NextDepends = *OldDepLast;
 315    *OldDepLast = Dep.Index();
 316    OldDepLast = &Dep->NextDepends;
 317
 318    return true;
 319 }
 320                                                                         /*}}}*/
 321 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* */
 324 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 325                                                 string PackageName,
 326                                                 string Version)
 327 {
 328    pkgCache &Cache = Owner->Cache;
 329
 330    // We do not add self referencing provides
 331    if (Ver.ParentPkg().Name() == PackageName)
 332       return true;
 333
 334    // Get a structure
 335    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 336    if (Provides == 0)
 337       return false;
 338    Cache.HeaderP->ProvidesCount++;
 339
 340    // Fill it in
 341    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 342    Prv->Version = Ver.Index();
 343    Prv->NextPkgProv = Ver->ProvidesList;
 344    Ver->ProvidesList = Prv.Index();
 345    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 346       return false;
 347
 348    // Locate the target package
 349    pkgCache::PkgIterator Pkg;
 350    if (Owner->NewPackage(Pkg,PackageName) == false)
 351       return false;
 352
 353    // Link it to the package
 354    Prv->ParentPkg = Pkg.Index();
 355    Prv->NextProvides = Pkg->ProvidesList;
 356    Pkg->ProvidesList = Prv.Index();
 357
 358    return true;
 359 }
 360                                                                         /*}}}*/
 361 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 362 // ---------------------------------------------------------------------
 363 /* This is used to select which file is to be associated with all newly
 364    added versions. */
 365 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 366 {
 367    struct stat Buf;
 368    if (stat(File.c_str(),&Buf) == -1)
 369       return _error->Errno("stat","Couldn't stat ",File.c_str());
 370
 371    // Get some space for the structure
 372    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 373    if (CurrentFile == Cache.PkgFileP)
 374       return false;
 375
 376    // Fill it in
 377    CurrentFile->FileName = Map.WriteString(File);
 378    CurrentFile->Size = Buf.st_size;
 379    CurrentFile->mtime = Buf.st_mtime;
 380    CurrentFile->NextFile = Cache.HeaderP->FileList;
 381    CurrentFile->Flags = Flags;
 382    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 383    PkgFileName = File;
 384    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 385    Cache.HeaderP->PackageFileCount++;
 386
 387    if (CurrentFile->FileName == 0)
 388       return false;
 389
 390    if (Progress != 0)
 391       Progress->SubProgress(Buf.st_size);
 392    return true;
 393 }
 394                                                                         /*}}}*/
 395 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 396 // ---------------------------------------------------------------------
 397 /* This is used to create handles to strings. Given the same text it
 398    always returns the same number */
 399 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 400                                                  unsigned int Size)
 401 {
 402    /* We use a very small transient hash table here, this speeds up generation
 403       by a fair amount on slower machines */
 404    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 405    if (Bucket != 0 &&
 406        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 407       return Bucket->String;
 408
 409    // Search for an insertion point
 410    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 411    int Res = 1;
 412    map_ptrloc *Last = &Cache.HeaderP->StringList;
 413    for (; I != Cache.StringItemP; Last = &I->NextItem,
 414         I = Cache.StringItemP + I->NextItem)
 415    {
 416       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 417       if (Res >= 0)
 418          break;
 419    }
 420
 421    // Match
 422    if (Res == 0)
 423    {
 424       Bucket = I;
 425       return I->String;
 426    }
 427
 428    // Get a structure
 429    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 430    if (Item == 0)
 431       return 0;
 432
 433    // Fill in the structure
 434    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 435    ItemP->NextItem = I - Cache.StringItemP;
 436    *Last = Item;
 437    ItemP->String = Map.WriteString(S,Size);
 438    if (ItemP->String == 0)
 439       return 0;
 440
 441    Bucket = ItemP;
 442    return ItemP->String;
 443 }
 444                                                                         /*}}}*/
 445
 446 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* The source cache is checked against the source list and the files
 449    on disk, any difference results in a false. */
 450 bool pkgSrcCacheCheck(pkgSourceList &List)
 451 {
 452    if (_error->PendingError() == true)
 453       return false;
 454
 455    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 456    string ListDir = _config->FindDir("Dir::State::lists");
 457
 458    // Count the number of missing files
 459    int Missing = 0;
 460    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 461    {
 462       // Only cache deb source types.
 463       if (I->Type != pkgSourceList::Item::Deb)
 464       {
 465          Missing++;
 466          continue;
 467       }
 468
 469       string File = ListDir + URItoFileName(I->PackagesURI());
 470       struct stat Buf;
 471       if (stat(File.c_str(),&Buf) != 0)
 472       {
 473          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 474                           I->PackagesInfo().c_str(),File.c_str());
 475          Missing++;
 476       }
 477    }
 478
 479    // Open the source package cache
 480    if (FileExists(CacheFile) == false)
 481       return false;
 482
 483    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 484    if (_error->PendingError() == true)
 485    {
 486       _error->Discard();
 487       return false;
 488    }
 489
 490    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 491    if (_error->PendingError() == true || Map.Size() == 0)
 492    {
 493       _error->Discard();
 494       return false;
 495    }
 496
 497    pkgCache Cache(Map);
 498    if (_error->PendingError() == true)
 499    {
 500       _error->Discard();
 501       return false;
 502    }
 503
 504    // They are certianly out of sync
 505    if (Cache.Head().PackageFileCount != List.size() - Missing)
 506       return false;
 507
 508    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 509    {
 510       // Search for a match in the source list
 511       bool Bad = true;
 512       for (pkgSourceList::const_iterator I = List.begin();
 513            I != List.end(); I++)
 514       {
 515          // Only cache deb source types.
 516          if (I->Type != pkgSourceList::Item::Deb)
 517             continue;
 518
 519          string File = ListDir + URItoFileName(I->PackagesURI());
 520          if (F.FileName() == File)
 521          {
 522             Bad = false;
 523             break;
 524          }
 525       }
 526
 527       // Check if the file matches what was cached
 528       Bad |= !F.IsOk();
 529       if (Bad == true)
 530          return false;
 531    }
 532
 533    return true;
 534 }
 535                                                                         /*}}}*/
 536 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 537 // ---------------------------------------------------------------------
 538 /* This does a simple check of all files used to compose the cache */
 539 bool pkgPkgCacheCheck(string CacheFile)
 540 {
 541    if (_error->PendingError() == true)
 542       return false;
 543
 544    // Open the source package cache
 545    if (FileExists(CacheFile) == false)
 546       return false;
 547
 548    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 549    if (_error->PendingError() == true)
 550    {
 551       _error->Discard();
 552       return false;
 553    }
 554
 555    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 556    if (_error->PendingError() == true || Map.Size() == 0)
 557    {
 558       _error->Discard();
 559       return false;
 560    }
 561
 562    pkgCache Cache(Map);
 563    if (_error->PendingError() == true)
 564    {
 565       _error->Discard();
 566       return false;
 567    }
 568
 569    // Status files that must be in the cache
 570    string Status[3];
 571    Status[0] = _config->FindFile("Dir::State::xstatus");
 572    Status[1]= _config->FindFile("Dir::State::userstatus");
 573    Status[2] = _config->FindFile("Dir::State::status");
 574
 575    // Cheack each file
 576    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 577    {
 578       if (F.IsOk() == false)
 579          return false;
 580
 581       // See if this is one of the status files
 582       for (int I = 0; I != 3; I++)
 583          if (F.FileName() == Status[I])
 584             Status[I] = string();
 585    }
 586
 587    // Make sure all the status files are loaded.
 588    for (int I = 0; I != 3; I++)
 589    {
 590       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 591          return false;
 592    }
 593
 594    return true;
 595 }
 596                                                                         /*}}}*/
 597 // AddStatusSize - Add the size of the status files                     /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* This adds the size of all the status files to the size counter */
 600 bool pkgAddStatusSize(unsigned long &TotalSize)
 601 {
 602    // Grab the file names
 603    string xstatus = _config->FindFile("Dir::State::xstatus");
 604    string userstatus = _config->FindFile("Dir::State::userstatus");
 605    string status = _config->FindFile("Dir::State::status");
 606
 607    // Grab the sizes
 608    struct stat Buf;
 609    if (stat(xstatus.c_str(),&Buf) == 0)
 610       TotalSize += Buf.st_size;
 611    if (stat(userstatus.c_str(),&Buf) == 0)
 612       TotalSize += Buf.st_size;
 613    if (stat(status.c_str(),&Buf) != 0)
 614       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 615    TotalSize += Buf.st_size;
 616
 617    return true;
 618 }
 619                                                                         /*}}}*/
 620 // MergeStatus - Add the status files to the cache                      /*{{{*/
 621 // ---------------------------------------------------------------------
 622 /* This adds the status files to the map */
 623 bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 624                     unsigned long &CurrentSize,unsigned long TotalSize)
 625 {
 626    // Grab the file names
 627    string Status[3];
 628    Status[0] = _config->FindFile("Dir::State::xstatus");
 629    Status[1]= _config->FindFile("Dir::State::userstatus");
 630    Status[2] = _config->FindFile("Dir::State::status");
 631
 632    for (int I = 0; I != 3; I++)
 633    {
 634       // Check if the file exists and it is not the primary status file.
 635       string File = Status[I];
 636       if (I != 2 && FileExists(File) == false)
 637          continue;
 638
 639       FileFd Pkg(File,FileFd::ReadOnly);
 640       debListParser Parser(Pkg);
 641       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 642       if (_error->PendingError() == true)
 643          return _error->Error("Problem opening %s",File.c_str());
 644       CurrentSize += Pkg.Size();
 645
 646       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 647       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 648          return _error->Error("Problem with SelectFile %s",File.c_str());
 649
 650       if (Gen.MergeList(Parser) == false)
 651          return _error->Error("Problem with MergeList %s",File.c_str());
 652       Progress.Progress(Pkg.Size());
 653    }
 654
 655    return true;
 656 }
 657                                                                         /*}}}*/
 658 // GenerateSrcCache - Write the source package lists to the map         /*{{{*/
 659 // ---------------------------------------------------------------------
 660 /* This puts the source package cache into the given generator. */
 661 bool pkgGenerateSrcCache(pkgSourceList &List,OpProgress &Progress,
 662                          pkgCacheGenerator &Gen,
 663                          unsigned long &CurrentSize,unsigned long &TotalSize)
 664 {
 665    string ListDir = _config->FindDir("Dir::State::lists");
 666
 667    // Prepare the progress indicator
 668    TotalSize = 0;
 669    struct stat Buf;
 670    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 671    {
 672       string File = ListDir + URItoFileName(I->PackagesURI());
 673       if (stat(File.c_str(),&Buf) != 0)
 674          continue;
 675       TotalSize += Buf.st_size;
 676    }
 677
 678    if (pkgAddStatusSize(TotalSize) == false)
 679       return false;
 680
 681    // Generate the pkg source cache
 682    CurrentSize = 0;
 683    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 684    {
 685       // Only cache deb source types.
 686       if (I->Type != pkgSourceList::Item::Deb)
 687          continue;
 688
 689       string File = ListDir + URItoFileName(I->PackagesURI());
 690
 691       if (FileExists(File) == false)
 692          continue;
 693
 694       FileFd Pkg(File,FileFd::ReadOnly);
 695       debListParser Parser(Pkg);
 696       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 697       if (_error->PendingError() == true)
 698          return _error->Error("Problem opening %s",File.c_str());
 699       CurrentSize += Pkg.Size();
 700
 701       Progress.SubProgress(0,I->PackagesInfo());
 702       if (Gen.SelectFile(File) == false)
 703          return _error->Error("Problem with SelectFile %s",File.c_str());
 704
 705       if (Gen.MergeList(Parser) == false)
 706          return _error->Error("Problem with MergeList %s",File.c_str());
 707
 708       // Check the release file
 709       string RFile = ListDir + URItoFileName(I->ReleaseURI());
 710       if (FileExists(RFile) == true)
 711       {
 712          FileFd Rel(RFile,FileFd::ReadOnly);
 713          if (_error->PendingError() == true)
 714             return false;
 715          Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 716       }
 717    }
 718
 719    return true;
 720 }
 721                                                                         /*}}}*/
 722 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 723 // ---------------------------------------------------------------------
 724 /* This copies the package source cache and then merges the status and
 725    xstatus files into it. */
 726 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 727 {
 728    unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
 729
 730    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 731
 732    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 733    bool SrcOk = pkgSrcCacheCheck(List);
 734    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 735
 736    // Rebuild the source and package caches
 737    if (SrcOk == false)
 738    {
 739       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 740       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 741       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 742       DynamicMMap Map(CacheF,MMap::Public,MapSize);
 743       if (_error->PendingError() == true)
 744          return false;
 745
 746       pkgCacheGenerator Gen(Map,Progress);
 747       unsigned long CurrentSize = 0;
 748       unsigned long TotalSize = 0;
 749       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 750          return false;
 751
 752       // Write the src cache
 753       Gen.GetCache().HeaderP->Dirty = false;
 754       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 755          return _error->Error("IO Error saving source cache");
 756       Gen.GetCache().HeaderP->Dirty = true;
 757
 758       // Merge in the source caches
 759       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 760    }
 761
 762    if (PkgOk == true)
 763    {
 764       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 765       return true;
 766    }
 767
 768    // We use the source cache to generate the package cache
 769    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 770
 771    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 772    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 773    DynamicMMap Map(CacheF,MMap::Public,MapSize);
 774    if (_error->PendingError() == true)
 775       return false;
 776
 777    // Preload the map with the source cache
 778    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 779                     SCacheF.Size()) == false)
 780       return false;
 781
 782    pkgCacheGenerator Gen(Map,Progress);
 783
 784    // Compute the progress
 785    unsigned long TotalSize = 0;
 786    if (pkgAddStatusSize(TotalSize) == false)
 787       return false;
 788
 789    unsigned long CurrentSize = 0;
 790    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 791 }
 792                                                                         /*}}}*/
 793 // MakeStatusCacheMem - Returns a map for the status cache              /*{{{*/
 794 // ---------------------------------------------------------------------
 795 /* This creates a map object for the status cache. If the process has write
 796    access to the caches then it is the same as MakeStatusCache, otherwise it
 797    creates a memory block and puts the cache in there. */
 798 MMap *pkgMakeStatusCacheMem(pkgSourceList &List,OpProgress &Progress)
 799 {
 800    unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
 801
 802    /* If the cache file is writeable this is just a wrapper for
 803       MakeStatusCache */
 804    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 805    bool Writeable = (access(CacheFile.c_str(),W_OK) == 0) ||
 806                     (errno == ENOENT);
 807
 808    if (Writeable == true)
 809    {
 810       if (pkgMakeStatusCache(List,Progress) == false)
 811          return 0;
 812
 813       // Open the cache file
 814       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 815       if (_error->PendingError() == true)
 816          return 0;
 817
 818       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 819       if (_error->PendingError() == true)
 820       {
 821          delete Map;
 822          return 0;
 823       }
 824       return Map;
 825    }
 826
 827    // Mostly from MakeStatusCache..
 828    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 829
 830    bool SrcOk = pkgSrcCacheCheck(List);
 831    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 832
 833    // Rebuild the source and package caches
 834    if (SrcOk == false)
 835    {
 836       DynamicMMap *Map = new DynamicMMap(MMap::Public,MapSize);
 837       if (_error->PendingError() == true)
 838       {
 839          delete Map;
 840          return 0;
 841       }
 842
 843       pkgCacheGenerator Gen(*Map,Progress);
 844       unsigned long CurrentSize = 0;
 845       unsigned long TotalSize = 0;
 846       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 847       {
 848          delete Map;
 849          return 0;
 850       }
 851
 852       // Merge in the source caches
 853       if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 854       {
 855          delete Map;
 856          return 0;
 857       }
 858
 859       return Map;
 860    }
 861
 862    if (PkgOk == true)
 863    {
 864       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 865
 866       // Open the cache file
 867       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 868       if (_error->PendingError() == true)
 869          return 0;
 870
 871       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 872       if (_error->PendingError() == true)
 873       {
 874          delete Map;
 875          return 0;
 876       }
 877       return Map;
 878    }
 879
 880    // We use the source cache to generate the package cache
 881    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 882    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 883    DynamicMMap *Map = new DynamicMMap(MMap::Public,MapSize);
 884    if (_error->PendingError() == true)
 885    {
 886       delete Map;
 887       return 0;
 888    }
 889
 890    // Preload the map with the source cache
 891    if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 892                     SCacheF.Size()) == false)
 893    {
 894       delete Map;
 895       return 0;
 896    }
 897
 898    pkgCacheGenerator Gen(*Map,Progress);
 899
 900    // Compute the progress
 901    unsigned long TotalSize = 0;
 902    if (pkgAddStatusSize(TotalSize) == false)
 903    {
 904       delete Map;
 905       return 0;
 906    }
 907
 908    unsigned long CurrentSize = 0;
 909    if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 910    {
 911       delete Map;
 912       return 0;
 913    }
 914
 915    return Map;
 916 }
 917                                                                         /*}}}*/