apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.41 1999/07/26 17:46:07 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25 #include <system.h>
  26
  27 #include <sys/stat.h>
  28 #include <unistd.h>
  29 #include <errno.h>
  30                                                                         /*}}}*/
  31
  32 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  33 // ---------------------------------------------------------------------
  34 /* We set the diry flag and make sure that is written to the disk */
  35 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  36                     Map(Map), Cache(Map), Progress(&Prog)
  37 {
  38    CurrentFile = 0;
  39
  40    if (_error->PendingError() == true)
  41       return;
  42
  43    if (Map.Size() == 0)
  44    {
  45       Map.RawAllocate(sizeof(pkgCache::Header));
  46       *Cache.HeaderP = pkgCache::Header();
  47    }
  48    Cache.HeaderP->Dirty = true;
  49    Map.Sync(0,sizeof(pkgCache::Header));
  50    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  51    memset(UniqHash,0,sizeof(UniqHash));
  52 }
  53                                                                         /*}}}*/
  54 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We sync the data then unset the dirty flag in two steps so as to
  57    advoid a problem during a crash */
  58 pkgCacheGenerator::~pkgCacheGenerator()
  59 {
  60    if (_error->PendingError() == true)
  61       return;
  62    if (Map.Sync() == false)
  63       return;
  64
  65    Cache.HeaderP->Dirty = false;
  66    Map.Sync(0,sizeof(pkgCache::Header));
  67 }
  68                                                                         /*}}}*/
  69 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  70 // ---------------------------------------------------------------------
  71 /* This provides the generation of the entries in the cache. Each loop
  72    goes through a single package record from the underlying parse engine. */
  73 bool pkgCacheGenerator::MergeList(ListParser &List,
  74                                   pkgCache::VerIterator *OutVer)
  75 {
  76    List.Owner = this;
  77
  78    unsigned int Counter = 0;
  79    while (List.Step() == true)
  80    {
  81       // Get a pointer to the package structure
  82       string PackageName = List.Package();
  83       if (PackageName.empty() == true)
  84          return false;
  85
  86       pkgCache::PkgIterator Pkg;
  87       if (NewPackage(Pkg,PackageName) == false)
  88          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  89       Counter++;
  90       if (Counter % 100 == 0 && Progress != 0)
  91          Progress->Progress(List.Offset());
  92
  93       /* Get a pointer to the version structure. We know the list is sorted
  94          so we use that fact in the search. Insertion of new versions is
  95          done with correct sorting */
  96       string Version = List.Version();
  97       if (Version.empty() == true)
  98       {
  99          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 100             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
 101          continue;
 102       }
 103
 104       pkgCache::VerIterator Ver = Pkg.VersionList();
 105       map_ptrloc *Last = &Pkg->VersionList;
 106       int Res = 1;
 107       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 108       {
 109          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 110                                  Ver.VerStr() + strlen(Ver.VerStr()));
 111          if (Res >= 0)
 112             break;
 113       }
 114
 115       /* We already have a version for this item, record that we
 116          saw it */
 117       unsigned long Hash = List.VersionHash();
 118       if (Res == 0 && Ver->Hash == Hash)
 119       {
 120          if (List.UsePackage(Pkg,Ver) == false)
 121             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 122
 123          if (NewFileVer(Ver,List) == false)
 124             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 125
 126          // Read only a single record and return
 127          if (OutVer != 0)
 128          {
 129             *OutVer = Ver;
 130             return true;
 131          }
 132
 133          continue;
 134       }
 135
 136       // Skip to the end of the same version set.
 137       if (Res == 0)
 138       {
 139          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 140          {
 141             Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 142                                     Ver.VerStr() + strlen(Ver.VerStr()));
 143             if (Res != 0)
 144                break;
 145          }
 146       }
 147
 148       // Add a new version
 149       *Last = NewVersion(Ver,Version,*Last);
 150       Ver->ParentPkg = Pkg.Index();
 151       Ver->Hash = Hash;
 152       if (List.NewVersion(Ver) == false)
 153          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 154
 155       if (List.UsePackage(Pkg,Ver) == false)
 156          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 157
 158       if (NewFileVer(Ver,List) == false)
 159          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 160
 161       // Read only a single record and return
 162       if (OutVer != 0)
 163       {
 164          *OutVer = Ver;
 165          return true;
 166       }
 167    }
 168
 169    return true;
 170 }
 171                                                                         /*}}}*/
 172 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 173 // ---------------------------------------------------------------------
 174 /* This creates a new package structure and adds it to the hash table */
 175 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 176 {
 177    Pkg = Cache.FindPkg(Name);
 178    if (Pkg.end() == false)
 179       return true;
 180
 181    // Get a structure
 182    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 183    if (Package == 0)
 184       return false;
 185
 186    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 187
 188    // Insert it into the hash table
 189    unsigned long Hash = Cache.Hash(Name);
 190    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 191    Cache.HeaderP->HashTable[Hash] = Package;
 192
 193    // Set the name and the ID
 194    Pkg->Name = Map.WriteString(Name);
 195    if (Pkg->Name == 0)
 196       return false;
 197    Pkg->ID = Cache.HeaderP->PackageCount++;
 198
 199    return true;
 200 }
 201                                                                         /*}}}*/
 202 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 203 // ---------------------------------------------------------------------
 204 /* */
 205 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 206                                    ListParser &List)
 207 {
 208    if (CurrentFile == 0)
 209       return true;
 210
 211    // Get a structure
 212    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 213    if (VerFile == 0)
 214       return 0;
 215
 216    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 217    VF->File = CurrentFile - Cache.PkgFileP;
 218
 219    // Link it to the end of the list
 220    map_ptrloc *Last = &Ver->FileList;
 221    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 222       Last = &V->NextFile;
 223    VF->NextFile = *Last;
 224    *Last = VF.Index();
 225
 226    VF->Offset = List.Offset();
 227    VF->Size = List.Size();
 228    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 229       Cache.HeaderP->MaxVerFileSize = VF->Size;
 230    Cache.HeaderP->VerFileCount++;
 231
 232    return true;
 233 }
 234                                                                         /*}}}*/
 235 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 236 // ---------------------------------------------------------------------
 237 /* This puts a version structure in the linked list */
 238 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 239                                             string VerStr,
 240                                             unsigned long Next)
 241 {
 242    // Get a structure
 243    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 244    if (Version == 0)
 245       return 0;
 246
 247    // Fill it in
 248    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 249    Ver->NextVer = Next;
 250    Ver->ID = Cache.HeaderP->VersionCount++;
 251    Ver->VerStr = Map.WriteString(VerStr);
 252    if (Ver->VerStr == 0)
 253       return 0;
 254
 255    return Version;
 256 }
 257                                                                         /*}}}*/
 258 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This creates a dependency element in the tree. It is linked to the
 261    version and to the package that it is pointing to. */
 262 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 263                                                string PackageName,
 264                                                string Version,
 265                                                unsigned int Op,
 266                                                unsigned int Type)
 267 {
 268    pkgCache &Cache = Owner->Cache;
 269
 270    // Get a structure
 271    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 272    if (Dependency == 0)
 273       return false;
 274
 275    // Fill it in
 276    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 277    Dep->ParentVer = Ver.Index();
 278    Dep->Type = Type;
 279    Dep->CompareOp = Op;
 280    Dep->ID = Cache.HeaderP->DependsCount++;
 281
 282    // Locate the target package
 283    pkgCache::PkgIterator Pkg;
 284    if (Owner->NewPackage(Pkg,PackageName) == false)
 285       return false;
 286
 287    // Probe the reverse dependency list for a version string that matches
 288    if (Version.empty() == false)
 289    {
 290 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++, Hit++)
 291          if (I->Version != 0 && I.TargetVer() == Version)
 292             Dep->Version = I->Version;*/
 293       if (Dep->Version == 0)
 294          if ((Dep->Version = WriteString(Version)) == 0)
 295             return false;
 296    }
 297
 298    // Link it to the package
 299    Dep->Package = Pkg.Index();
 300    Dep->NextRevDepends = Pkg->RevDepends;
 301    Pkg->RevDepends = Dep.Index();
 302
 303    /* Link it to the version (at the end of the list)
 304       Caching the old end point speeds up generation substantially */
 305    if (OldDepVer != Ver)
 306    {
 307       OldDepLast = &Ver->DependsList;
 308       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 309          OldDepLast = &D->NextDepends;
 310       OldDepVer = Ver;
 311    }
 312
 313    Dep->NextDepends = *OldDepLast;
 314    *OldDepLast = Dep.Index();
 315    OldDepLast = &Dep->NextDepends;
 316
 317    return true;
 318 }
 319                                                                         /*}}}*/
 320 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 321 // ---------------------------------------------------------------------
 322 /* */
 323 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 324                                                 string PackageName,
 325                                                 string Version)
 326 {
 327    pkgCache &Cache = Owner->Cache;
 328
 329    // We do not add self referencing provides
 330    if (Ver.ParentPkg().Name() == PackageName)
 331       return true;
 332
 333    // Get a structure
 334    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 335    if (Provides == 0)
 336       return false;
 337    Cache.HeaderP->ProvidesCount++;
 338
 339    // Fill it in
 340    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 341    Prv->Version = Ver.Index();
 342    Prv->NextPkgProv = Ver->ProvidesList;
 343    Ver->ProvidesList = Prv.Index();
 344    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 345       return false;
 346
 347    // Locate the target package
 348    pkgCache::PkgIterator Pkg;
 349    if (Owner->NewPackage(Pkg,PackageName) == false)
 350       return false;
 351
 352    // Link it to the package
 353    Prv->ParentPkg = Pkg.Index();
 354    Prv->NextProvides = Pkg->ProvidesList;
 355    Pkg->ProvidesList = Prv.Index();
 356
 357    return true;
 358 }
 359                                                                         /*}}}*/
 360 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 361 // ---------------------------------------------------------------------
 362 /* This is used to select which file is to be associated with all newly
 363    added versions. */
 364 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 365 {
 366    struct stat Buf;
 367    if (stat(File.c_str(),&Buf) == -1)
 368       return _error->Errno("stat","Couldn't stat ",File.c_str());
 369
 370    // Get some space for the structure
 371    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 372    if (CurrentFile == Cache.PkgFileP)
 373       return false;
 374
 375    // Fill it in
 376    CurrentFile->FileName = Map.WriteString(File);
 377    CurrentFile->Size = Buf.st_size;
 378    CurrentFile->mtime = Buf.st_mtime;
 379    CurrentFile->NextFile = Cache.HeaderP->FileList;
 380    CurrentFile->Flags = Flags;
 381    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 382    PkgFileName = File;
 383    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 384    Cache.HeaderP->PackageFileCount++;
 385
 386    if (CurrentFile->FileName == 0)
 387       return false;
 388
 389    if (Progress != 0)
 390       Progress->SubProgress(Buf.st_size);
 391    return true;
 392 }
 393                                                                         /*}}}*/
 394 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 395 // ---------------------------------------------------------------------
 396 /* This is used to create handles to strings. Given the same text it
 397    always returns the same number */
 398 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 399                                                  unsigned int Size)
 400 {
 401    /* We use a very small transient hash table here, this speeds up generation
 402       by a fair amount on slower machines */
 403    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 404    if (Bucket != 0 &&
 405        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 406       return Bucket->String;
 407
 408    // Search for an insertion point
 409    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 410    int Res = 1;
 411    map_ptrloc *Last = &Cache.HeaderP->StringList;
 412    for (; I != Cache.StringItemP; Last = &I->NextItem,
 413         I = Cache.StringItemP + I->NextItem)
 414    {
 415       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 416       if (Res >= 0)
 417          break;
 418    }
 419
 420    // Match
 421    if (Res == 0)
 422    {
 423       Bucket = I;
 424       return I->String;
 425    }
 426
 427    // Get a structure
 428    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 429    if (Item == 0)
 430       return 0;
 431
 432    // Fill in the structure
 433    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 434    ItemP->NextItem = I - Cache.StringItemP;
 435    *Last = Item;
 436    ItemP->String = Map.WriteString(S,Size);
 437    if (ItemP->String == 0)
 438       return 0;
 439
 440    Bucket = ItemP;
 441    return ItemP->String;
 442 }
 443                                                                         /*}}}*/
 444
 445 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 446 // ---------------------------------------------------------------------
 447 /* The source cache is checked against the source list and the files
 448    on disk, any difference results in a false. */
 449 bool pkgSrcCacheCheck(pkgSourceList &List)
 450 {
 451    if (_error->PendingError() == true)
 452       return false;
 453
 454    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 455    string ListDir = _config->FindDir("Dir::State::lists");
 456
 457    // Count the number of missing files
 458    int Missing = 0;
 459    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 460    {
 461       // Only cache deb source types.
 462       if (I->Type != pkgSourceList::Item::Deb)
 463       {
 464          Missing++;
 465          continue;
 466       }
 467
 468       string File = ListDir + URItoFileName(I->PackagesURI());
 469       struct stat Buf;
 470       if (stat(File.c_str(),&Buf) != 0)
 471       {
 472          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 473                           I->PackagesInfo().c_str(),File.c_str());
 474          Missing++;
 475       }
 476    }
 477
 478    // Open the source package cache
 479    if (FileExists(CacheFile) == false)
 480       return false;
 481
 482    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 483    if (_error->PendingError() == true)
 484    {
 485       _error->Discard();
 486       return false;
 487    }
 488
 489    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 490    if (_error->PendingError() == true || Map.Size() == 0)
 491    {
 492       _error->Discard();
 493       return false;
 494    }
 495
 496    pkgCache Cache(Map);
 497    if (_error->PendingError() == true)
 498    {
 499       _error->Discard();
 500       return false;
 501    }
 502
 503    // They are certianly out of sync
 504    if (Cache.Head().PackageFileCount != List.size() - Missing)
 505       return false;
 506
 507    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 508    {
 509       // Search for a match in the source list
 510       bool Bad = true;
 511       for (pkgSourceList::const_iterator I = List.begin();
 512            I != List.end(); I++)
 513       {
 514          // Only cache deb source types.
 515          if (I->Type != pkgSourceList::Item::Deb)
 516             continue;
 517
 518          string File = ListDir + URItoFileName(I->PackagesURI());
 519          if (F.FileName() == File)
 520          {
 521             Bad = false;
 522             break;
 523          }
 524       }
 525
 526       // Check if the file matches what was cached
 527       Bad |= !F.IsOk();
 528       if (Bad == true)
 529          return false;
 530    }
 531
 532    return true;
 533 }
 534                                                                         /*}}}*/
 535 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 536 // ---------------------------------------------------------------------
 537 /* This does a simple check of all files used to compose the cache */
 538 bool pkgPkgCacheCheck(string CacheFile)
 539 {
 540    if (_error->PendingError() == true)
 541       return false;
 542
 543    // Open the source package cache
 544    if (FileExists(CacheFile) == false)
 545       return false;
 546
 547    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 548    if (_error->PendingError() == true)
 549    {
 550       _error->Discard();
 551       return false;
 552    }
 553
 554    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 555    if (_error->PendingError() == true || Map.Size() == 0)
 556    {
 557       _error->Discard();
 558       return false;
 559    }
 560
 561    pkgCache Cache(Map);
 562    if (_error->PendingError() == true)
 563    {
 564       _error->Discard();
 565       return false;
 566    }
 567
 568    // Status files that must be in the cache
 569    string Status[3];
 570    Status[0] = _config->FindFile("Dir::State::xstatus");
 571    Status[1]= _config->FindFile("Dir::State::userstatus");
 572    Status[2] = _config->FindFile("Dir::State::status");
 573
 574    // Cheack each file
 575    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 576    {
 577       if (F.IsOk() == false)
 578          return false;
 579
 580       // See if this is one of the status files
 581       for (int I = 0; I != 3; I++)
 582          if (F.FileName() == Status[I])
 583             Status[I] = string();
 584    }
 585
 586    // Make sure all the status files are loaded.
 587    for (int I = 0; I != 3; I++)
 588    {
 589       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 590          return false;
 591    }
 592
 593    return true;
 594 }
 595                                                                         /*}}}*/
 596 // AddStatusSize - Add the size of the status files                     /*{{{*/
 597 // ---------------------------------------------------------------------
 598 /* This adds the size of all the status files to the size counter */
 599 bool pkgAddStatusSize(unsigned long &TotalSize)
 600 {
 601    // Grab the file names
 602    string xstatus = _config->FindFile("Dir::State::xstatus");
 603    string userstatus = _config->FindFile("Dir::State::userstatus");
 604    string status = _config->FindFile("Dir::State::status");
 605
 606    // Grab the sizes
 607    struct stat Buf;
 608    if (stat(xstatus.c_str(),&Buf) == 0)
 609       TotalSize += Buf.st_size;
 610    if (stat(userstatus.c_str(),&Buf) == 0)
 611       TotalSize += Buf.st_size;
 612    if (stat(status.c_str(),&Buf) != 0)
 613       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 614    TotalSize += Buf.st_size;
 615
 616    return true;
 617 }
 618                                                                         /*}}}*/
 619 // MergeStatus - Add the status files to the cache                      /*{{{*/
 620 // ---------------------------------------------------------------------
 621 /* This adds the status files to the map */
 622 bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 623                     unsigned long &CurrentSize,unsigned long TotalSize)
 624 {
 625    // Grab the file names
 626    string Status[3];
 627    Status[0] = _config->FindFile("Dir::State::xstatus");
 628    Status[1]= _config->FindFile("Dir::State::userstatus");
 629    Status[2] = _config->FindFile("Dir::State::status");
 630
 631    for (int I = 0; I != 3; I++)
 632    {
 633       // Check if the file exists and it is not the primary status file.
 634       string File = Status[I];
 635       if (I != 2 && FileExists(File) == false)
 636          continue;
 637
 638       FileFd Pkg(File,FileFd::ReadOnly);
 639       debListParser Parser(Pkg);
 640       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 641       if (_error->PendingError() == true)
 642          return _error->Error("Problem opening %s",File.c_str());
 643       CurrentSize += Pkg.Size();
 644
 645       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 646       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 647          return _error->Error("Problem with SelectFile %s",File.c_str());
 648
 649       if (Gen.MergeList(Parser) == false)
 650          return _error->Error("Problem with MergeList %s",File.c_str());
 651       Progress.Progress(Pkg.Size());
 652    }
 653
 654    return true;
 655 }
 656                                                                         /*}}}*/
 657 // GenerateSrcCache - Write the source package lists to the map         /*{{{*/
 658 // ---------------------------------------------------------------------
 659 /* This puts the source package cache into the given generator. */
 660 bool pkgGenerateSrcCache(pkgSourceList &List,OpProgress &Progress,
 661                          pkgCacheGenerator &Gen,
 662                          unsigned long &CurrentSize,unsigned long &TotalSize)
 663 {
 664    string ListDir = _config->FindDir("Dir::State::lists");
 665
 666    // Prepare the progress indicator
 667    TotalSize = 0;
 668    struct stat Buf;
 669    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 670    {
 671       string File = ListDir + URItoFileName(I->PackagesURI());
 672       if (stat(File.c_str(),&Buf) != 0)
 673          continue;
 674       TotalSize += Buf.st_size;
 675    }
 676
 677    if (pkgAddStatusSize(TotalSize) == false)
 678       return false;
 679
 680    // Generate the pkg source cache
 681    CurrentSize = 0;
 682    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 683    {
 684       // Only cache deb source types.
 685       if (I->Type != pkgSourceList::Item::Deb)
 686          continue;
 687
 688       string File = ListDir + URItoFileName(I->PackagesURI());
 689
 690       if (FileExists(File) == false)
 691          continue;
 692
 693       FileFd Pkg(File,FileFd::ReadOnly);
 694       debListParser Parser(Pkg);
 695       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 696       if (_error->PendingError() == true)
 697          return _error->Error("Problem opening %s",File.c_str());
 698       CurrentSize += Pkg.Size();
 699
 700       Progress.SubProgress(0,I->PackagesInfo());
 701       if (Gen.SelectFile(File) == false)
 702          return _error->Error("Problem with SelectFile %s",File.c_str());
 703
 704       if (Gen.MergeList(Parser) == false)
 705          return _error->Error("Problem with MergeList %s",File.c_str());
 706
 707       // Check the release file
 708       string RFile = ListDir + URItoFileName(I->ReleaseURI());
 709       if (FileExists(RFile) == true)
 710       {
 711          FileFd Rel(RFile,FileFd::ReadOnly);
 712          if (_error->PendingError() == true)
 713             return false;
 714          Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 715       }
 716    }
 717
 718    return true;
 719 }
 720                                                                         /*}}}*/
 721 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 722 // ---------------------------------------------------------------------
 723 /* This copies the package source cache and then merges the status and
 724    xstatus files into it. */
 725 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 726 {
 727    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 728
 729    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 730    bool SrcOk = pkgSrcCacheCheck(List);
 731    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 732
 733    // Rebuild the source and package caches
 734    if (SrcOk == false)
 735    {
 736       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 737       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 738       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 739       DynamicMMap Map(CacheF,MMap::Public);
 740       if (_error->PendingError() == true)
 741          return false;
 742
 743       pkgCacheGenerator Gen(Map,Progress);
 744       unsigned long CurrentSize = 0;
 745       unsigned long TotalSize = 0;
 746       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 747          return false;
 748
 749       // Write the src cache
 750       Gen.GetCache().HeaderP->Dirty = false;
 751       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 752          return _error->Error("IO Error saving source cache");
 753       Gen.GetCache().HeaderP->Dirty = true;
 754
 755       // Merge in the source caches
 756       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 757    }
 758
 759    if (PkgOk == true)
 760    {
 761       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 762       return true;
 763    }
 764
 765    // We use the source cache to generate the package cache
 766    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 767
 768    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 769    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 770    DynamicMMap Map(CacheF,MMap::Public);
 771    if (_error->PendingError() == true)
 772       return false;
 773
 774    // Preload the map with the source cache
 775    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 776                     SCacheF.Size()) == false)
 777       return false;
 778
 779    pkgCacheGenerator Gen(Map,Progress);
 780
 781    // Compute the progress
 782    unsigned long TotalSize = 0;
 783    if (pkgAddStatusSize(TotalSize) == false)
 784       return false;
 785
 786    unsigned long CurrentSize = 0;
 787    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 788 }
 789                                                                         /*}}}*/
 790 // MakeStatusCacheMem - Returns a map for the status cache              /*{{{*/
 791 // ---------------------------------------------------------------------
 792 /* This creates a map object for the status cache. If the process has write
 793    access to the caches then it is the same as MakeStatusCache, otherwise it
 794    creates a memory block and puts the cache in there. */
 795 MMap *pkgMakeStatusCacheMem(pkgSourceList &List,OpProgress &Progress)
 796 {
 797    /* If the cache file is writeable this is just a wrapper for
 798       MakeStatusCache */
 799    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 800    bool Writeable = (access(CacheFile.c_str(),W_OK) == 0) ||
 801                     (errno == ENOENT);
 802
 803    if (Writeable == true)
 804    {
 805       if (pkgMakeStatusCache(List,Progress) == false)
 806          return 0;
 807
 808       // Open the cache file
 809       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 810       if (_error->PendingError() == true)
 811          return 0;
 812
 813       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 814       if (_error->PendingError() == true)
 815       {
 816          delete Map;
 817          return 0;
 818       }
 819       return Map;
 820    }
 821
 822    // Mostly from MakeStatusCache..
 823    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 824
 825    bool SrcOk = pkgSrcCacheCheck(List);
 826    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 827
 828    // Rebuild the source and package caches
 829    if (SrcOk == false)
 830    {
 831       DynamicMMap *Map = new DynamicMMap(MMap::Public);
 832       if (_error->PendingError() == true)
 833       {
 834          delete Map;
 835          return 0;
 836       }
 837
 838       pkgCacheGenerator Gen(*Map,Progress);
 839       unsigned long CurrentSize = 0;
 840       unsigned long TotalSize = 0;
 841       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 842       {
 843          delete Map;
 844          return 0;
 845       }
 846
 847       // Merge in the source caches
 848       if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 849       {
 850          delete Map;
 851          return 0;
 852       }
 853
 854       return Map;
 855    }
 856
 857    if (PkgOk == true)
 858    {
 859       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 860
 861       // Open the cache file
 862       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 863       if (_error->PendingError() == true)
 864          return 0;
 865
 866       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 867       if (_error->PendingError() == true)
 868       {
 869          delete Map;
 870          return 0;
 871       }
 872       return Map;
 873    }
 874
 875    // We use the source cache to generate the package cache
 876    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 877    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 878    DynamicMMap *Map = new DynamicMMap(MMap::Public);
 879    if (_error->PendingError() == true)
 880    {
 881       delete Map;
 882       return 0;
 883    }
 884
 885    // Preload the map with the source cache
 886    if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 887                     SCacheF.Size()) == false)
 888    {
 889       delete Map;
 890       return 0;
 891    }
 892
 893    pkgCacheGenerator Gen(*Map,Progress);
 894
 895    // Compute the progress
 896    unsigned long TotalSize = 0;
 897    if (pkgAddStatusSize(TotalSize) == false)
 898    {
 899       delete Map;
 900       return 0;
 901    }
 902
 903    unsigned long CurrentSize = 0;
 904    if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 905    {
 906       delete Map;
 907       return 0;
 908    }
 909
 910    return Map;
 911 }
 912                                                                         /*}}}*/