apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.45 2000/01/14 06:26:36 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25
  26 #include <sys/stat.h>
  27 #include <unistd.h>
  28 #include <errno.h>
  29 #include <stdio.h>
  30 #include <system.h>
  31                                                                         /*}}}*/
  32
  33 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  34 // ---------------------------------------------------------------------
  35 /* We set the diry flag and make sure that is written to the disk */
  36 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  37                     Map(Map), Cache(Map), Progress(&Prog)
  38 {
  39    CurrentFile = 0;
  40
  41    if (_error->PendingError() == true)
  42       return;
  43
  44    if (Map.Size() == 0)
  45    {
  46       Map.RawAllocate(sizeof(pkgCache::Header));
  47       *Cache.HeaderP = pkgCache::Header();
  48    }
  49    Cache.HeaderP->Dirty = true;
  50    Map.Sync(0,sizeof(pkgCache::Header));
  51    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  52    memset(UniqHash,0,sizeof(UniqHash));
  53 }
  54                                                                         /*}}}*/
  55 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* We sync the data then unset the dirty flag in two steps so as to
  58    advoid a problem during a crash */
  59 pkgCacheGenerator::~pkgCacheGenerator()
  60 {
  61    if (_error->PendingError() == true)
  62       return;
  63    if (Map.Sync() == false)
  64       return;
  65
  66    Cache.HeaderP->Dirty = false;
  67    Map.Sync(0,sizeof(pkgCache::Header));
  68 }
  69                                                                         /*}}}*/
  70 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  71 // ---------------------------------------------------------------------
  72 /* This provides the generation of the entries in the cache. Each loop
  73    goes through a single package record from the underlying parse engine. */
  74 bool pkgCacheGenerator::MergeList(ListParser &List,
  75                                   pkgCache::VerIterator *OutVer)
  76 {
  77    List.Owner = this;
  78
  79    unsigned int Counter = 0;
  80    while (List.Step() == true)
  81    {
  82       // Get a pointer to the package structure
  83       string PackageName = List.Package();
  84       if (PackageName.empty() == true)
  85          return false;
  86
  87       pkgCache::PkgIterator Pkg;
  88       if (NewPackage(Pkg,PackageName) == false)
  89          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  90       Counter++;
  91       if (Counter % 100 == 0 && Progress != 0)
  92          Progress->Progress(List.Offset());
  93
  94       /* Get a pointer to the version structure. We know the list is sorted
  95          so we use that fact in the search. Insertion of new versions is
  96          done with correct sorting */
  97       string Version = List.Version();
  98       if (Version.empty() == true)
  99       {
 100          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 101             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
 102          continue;
 103       }
 104
 105       pkgCache::VerIterator Ver = Pkg.VersionList();
 106       map_ptrloc *Last = &Pkg->VersionList;
 107       int Res = 1;
 108       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 109       {
 110          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 111                                  Ver.VerStr() + strlen(Ver.VerStr()));
 112          if (Res >= 0)
 113             break;
 114       }
 115
 116       /* We already have a version for this item, record that we
 117          saw it */
 118       unsigned long Hash = List.VersionHash();
 119       if (Res == 0 && Ver->Hash == Hash)
 120       {
 121          if (List.UsePackage(Pkg,Ver) == false)
 122             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 123
 124          if (NewFileVer(Ver,List) == false)
 125             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 126
 127          // Read only a single record and return
 128          if (OutVer != 0)
 129          {
 130             *OutVer = Ver;
 131             return true;
 132          }
 133
 134          continue;
 135       }
 136
 137       // Skip to the end of the same version set.
 138       if (Res == 0)
 139       {
 140          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 141          {
 142             Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 143                                     Ver.VerStr() + strlen(Ver.VerStr()));
 144             if (Res != 0)
 145                break;
 146          }
 147       }
 148
 149       // Add a new version
 150       *Last = NewVersion(Ver,Version,*Last);
 151       Ver->ParentPkg = Pkg.Index();
 152       Ver->Hash = Hash;
 153       if (List.NewVersion(Ver) == false)
 154          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 155
 156       if (List.UsePackage(Pkg,Ver) == false)
 157          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 158
 159       if (NewFileVer(Ver,List) == false)
 160          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 161
 162       // Read only a single record and return
 163       if (OutVer != 0)
 164       {
 165          *OutVer = Ver;
 166          return true;
 167       }
 168    }
 169
 170    return true;
 171 }
 172                                                                         /*}}}*/
 173 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 174 // ---------------------------------------------------------------------
 175 /* This creates a new package structure and adds it to the hash table */
 176 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 177 {
 178    Pkg = Cache.FindPkg(Name);
 179    if (Pkg.end() == false)
 180       return true;
 181
 182    // Get a structure
 183    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 184    if (Package == 0)
 185       return false;
 186
 187    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 188
 189    // Insert it into the hash table
 190    unsigned long Hash = Cache.Hash(Name);
 191    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 192    Cache.HeaderP->HashTable[Hash] = Package;
 193
 194    // Set the name and the ID
 195    Pkg->Name = Map.WriteString(Name);
 196    if (Pkg->Name == 0)
 197       return false;
 198    Pkg->ID = Cache.HeaderP->PackageCount++;
 199
 200    return true;
 201 }
 202                                                                         /*}}}*/
 203 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 204 // ---------------------------------------------------------------------
 205 /* */
 206 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 207                                    ListParser &List)
 208 {
 209    if (CurrentFile == 0)
 210       return true;
 211
 212    // Get a structure
 213    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 214    if (VerFile == 0)
 215       return 0;
 216
 217    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 218    VF->File = CurrentFile - Cache.PkgFileP;
 219
 220    // Link it to the end of the list
 221    map_ptrloc *Last = &Ver->FileList;
 222    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 223       Last = &V->NextFile;
 224    VF->NextFile = *Last;
 225    *Last = VF.Index();
 226
 227    VF->Offset = List.Offset();
 228    VF->Size = List.Size();
 229    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 230       Cache.HeaderP->MaxVerFileSize = VF->Size;
 231    Cache.HeaderP->VerFileCount++;
 232
 233    return true;
 234 }
 235                                                                         /*}}}*/
 236 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 237 // ---------------------------------------------------------------------
 238 /* This puts a version structure in the linked list */
 239 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 240                                             string VerStr,
 241                                             unsigned long Next)
 242 {
 243    // Get a structure
 244    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 245    if (Version == 0)
 246       return 0;
 247
 248    // Fill it in
 249    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 250    Ver->NextVer = Next;
 251    Ver->ID = Cache.HeaderP->VersionCount++;
 252    Ver->VerStr = Map.WriteString(VerStr);
 253    if (Ver->VerStr == 0)
 254       return 0;
 255
 256    return Version;
 257 }
 258                                                                         /*}}}*/
 259 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 260 // ---------------------------------------------------------------------
 261 /* This creates a dependency element in the tree. It is linked to the
 262    version and to the package that it is pointing to. */
 263 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 264                                                string PackageName,
 265                                                string Version,
 266                                                unsigned int Op,
 267                                                unsigned int Type)
 268 {
 269    pkgCache &Cache = Owner->Cache;
 270
 271    // Get a structure
 272    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 273    if (Dependency == 0)
 274       return false;
 275
 276    // Fill it in
 277    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 278    Dep->ParentVer = Ver.Index();
 279    Dep->Type = Type;
 280    Dep->CompareOp = Op;
 281    Dep->ID = Cache.HeaderP->DependsCount++;
 282
 283    // Locate the target package
 284    pkgCache::PkgIterator Pkg;
 285    if (Owner->NewPackage(Pkg,PackageName) == false)
 286       return false;
 287
 288    // Probe the reverse dependency list for a version string that matches
 289    if (Version.empty() == false)
 290    {
 291 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++, Hit++)
 292          if (I->Version != 0 && I.TargetVer() == Version)
 293             Dep->Version = I->Version;*/
 294       if (Dep->Version == 0)
 295          if ((Dep->Version = WriteString(Version)) == 0)
 296             return false;
 297    }
 298
 299    // Link it to the package
 300    Dep->Package = Pkg.Index();
 301    Dep->NextRevDepends = Pkg->RevDepends;
 302    Pkg->RevDepends = Dep.Index();
 303
 304    /* Link it to the version (at the end of the list)
 305       Caching the old end point speeds up generation substantially */
 306    if (OldDepVer != Ver)
 307    {
 308       OldDepLast = &Ver->DependsList;
 309       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 310          OldDepLast = &D->NextDepends;
 311       OldDepVer = Ver;
 312    }
 313
 314    Dep->NextDepends = *OldDepLast;
 315    *OldDepLast = Dep.Index();
 316    OldDepLast = &Dep->NextDepends;
 317
 318    return true;
 319 }
 320                                                                         /*}}}*/
 321 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* */
 324 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 325                                                 string PackageName,
 326                                                 string Version)
 327 {
 328    pkgCache &Cache = Owner->Cache;
 329
 330    // We do not add self referencing provides
 331    if (Ver.ParentPkg().Name() == PackageName)
 332       return true;
 333
 334    // Get a structure
 335    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 336    if (Provides == 0)
 337       return false;
 338    Cache.HeaderP->ProvidesCount++;
 339
 340    // Fill it in
 341    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 342    Prv->Version = Ver.Index();
 343    Prv->NextPkgProv = Ver->ProvidesList;
 344    Ver->ProvidesList = Prv.Index();
 345    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 346       return false;
 347
 348    // Locate the target package
 349    pkgCache::PkgIterator Pkg;
 350    if (Owner->NewPackage(Pkg,PackageName) == false)
 351       return false;
 352
 353    // Link it to the package
 354    Prv->ParentPkg = Pkg.Index();
 355    Prv->NextProvides = Pkg->ProvidesList;
 356    Pkg->ProvidesList = Prv.Index();
 357
 358    return true;
 359 }
 360                                                                         /*}}}*/
 361 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 362 // ---------------------------------------------------------------------
 363 /* This is used to select which file is to be associated with all newly
 364    added versions. */
 365 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 366 {
 367    struct stat Buf;
 368    if (stat(File.c_str(),&Buf) == -1)
 369       return _error->Errno("stat","Couldn't stat ",File.c_str());
 370
 371    // Get some space for the structure
 372    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 373    if (CurrentFile == Cache.PkgFileP)
 374       return false;
 375
 376    // Fill it in
 377    CurrentFile->FileName = Map.WriteString(File);
 378    CurrentFile->Size = Buf.st_size;
 379    CurrentFile->mtime = Buf.st_mtime;
 380    CurrentFile->NextFile = Cache.HeaderP->FileList;
 381    CurrentFile->Flags = Flags;
 382    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 383    PkgFileName = File;
 384    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 385    Cache.HeaderP->PackageFileCount++;
 386
 387    if (CurrentFile->FileName == 0)
 388       return false;
 389
 390    if (Progress != 0)
 391       Progress->SubProgress(Buf.st_size);
 392    return true;
 393 }
 394                                                                         /*}}}*/
 395 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 396 // ---------------------------------------------------------------------
 397 /* This is used to create handles to strings. Given the same text it
 398    always returns the same number */
 399 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 400                                                  unsigned int Size)
 401 {
 402    /* We use a very small transient hash table here, this speeds up generation
 403       by a fair amount on slower machines */
 404    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 405    if (Bucket != 0 &&
 406        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 407       return Bucket->String;
 408
 409    // Search for an insertion point
 410    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 411    int Res = 1;
 412    map_ptrloc *Last = &Cache.HeaderP->StringList;
 413    for (; I != Cache.StringItemP; Last = &I->NextItem,
 414         I = Cache.StringItemP + I->NextItem)
 415    {
 416       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 417       if (Res >= 0)
 418          break;
 419    }
 420
 421    // Match
 422    if (Res == 0)
 423    {
 424       Bucket = I;
 425       return I->String;
 426    }
 427
 428    // Get a structure
 429    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 430    if (Item == 0)
 431       return 0;
 432
 433    // Fill in the structure
 434    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 435    ItemP->NextItem = I - Cache.StringItemP;
 436    *Last = Item;
 437    ItemP->String = Map.WriteString(S,Size);
 438    if (ItemP->String == 0)
 439       return 0;
 440
 441    Bucket = ItemP;
 442    return ItemP->String;
 443 }
 444                                                                         /*}}}*/
 445
 446 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* The source cache is checked against the source list and the files
 449    on disk, any difference results in a false. */
 450 bool pkgSrcCacheCheck(pkgSourceList &List)
 451 {
 452    if (_error->PendingError() == true)
 453       return false;
 454
 455    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 456    string ListDir = _config->FindDir("Dir::State::lists");
 457
 458    // Count the number of missing files
 459    int Missing = 0;
 460    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 461    {
 462       // Only cache deb source types.
 463       if (I->Type != pkgSourceList::Item::Deb)
 464       {
 465          Missing++;
 466          continue;
 467       }
 468
 469       string File = ListDir + URItoFileName(I->PackagesURI());
 470       struct stat Buf;
 471       if (stat(File.c_str(),&Buf) != 0)
 472       {
 473          // Old format file name.. rename it
 474          if (File[0] == '_' && stat(File.c_str()+1,&Buf) == 0)
 475          {
 476             if (rename(File.c_str()+1,File.c_str()) != 0)
 477                return _error->Errno("rename","Failed to rename %s to %s",
 478                                     File.c_str()+1,File.c_str());
 479             continue;
 480          }
 481
 482          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 483                           I->PackagesInfo().c_str(),File.c_str());
 484          Missing++;
 485       }
 486    }
 487
 488    // Open the source package cache
 489    if (FileExists(CacheFile) == false)
 490       return false;
 491
 492    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 493    if (_error->PendingError() == true)
 494    {
 495       _error->Discard();
 496       return false;
 497    }
 498
 499    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 500    if (_error->PendingError() == true || Map.Size() == 0)
 501    {
 502       _error->Discard();
 503       return false;
 504    }
 505
 506    pkgCache Cache(Map);
 507    if (_error->PendingError() == true)
 508    {
 509       _error->Discard();
 510       return false;
 511    }
 512
 513    // They are certianly out of sync
 514    if (Cache.Head().PackageFileCount != List.size() - Missing)
 515       return false;
 516
 517    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 518    {
 519       // Search for a match in the source list
 520       bool Bad = true;
 521       for (pkgSourceList::const_iterator I = List.begin();
 522            I != List.end(); I++)
 523       {
 524          // Only cache deb source types.
 525          if (I->Type != pkgSourceList::Item::Deb)
 526             continue;
 527
 528          string File = ListDir + URItoFileName(I->PackagesURI());
 529          if (F.FileName() == File)
 530          {
 531             Bad = false;
 532             break;
 533          }
 534       }
 535
 536       // Check if the file matches what was cached
 537       Bad |= !F.IsOk();
 538       if (Bad == true)
 539          return false;
 540    }
 541
 542    return true;
 543 }
 544                                                                         /*}}}*/
 545 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 546 // ---------------------------------------------------------------------
 547 /* This does a simple check of all files used to compose the cache */
 548 bool pkgPkgCacheCheck(string CacheFile)
 549 {
 550    if (_error->PendingError() == true)
 551       return false;
 552
 553    // Open the source package cache
 554    if (FileExists(CacheFile) == false)
 555       return false;
 556
 557    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 558    if (_error->PendingError() == true)
 559    {
 560       _error->Discard();
 561       return false;
 562    }
 563
 564    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 565    if (_error->PendingError() == true || Map.Size() == 0)
 566    {
 567       _error->Discard();
 568       return false;
 569    }
 570
 571    pkgCache Cache(Map);
 572    if (_error->PendingError() == true)
 573    {
 574       _error->Discard();
 575       return false;
 576    }
 577
 578    // Status files that must be in the cache
 579    string Status[3];
 580    Status[0] = _config->FindFile("Dir::State::xstatus");
 581    Status[1]= _config->FindFile("Dir::State::userstatus");
 582    Status[2] = _config->FindFile("Dir::State::status");
 583
 584    // Cheack each file
 585    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 586    {
 587       if (F.IsOk() == false)
 588          return false;
 589
 590       // See if this is one of the status files
 591       for (int I = 0; I != 3; I++)
 592          if (F.FileName() == Status[I])
 593             Status[I] = string();
 594    }
 595
 596    // Make sure all the status files are loaded.
 597    for (int I = 0; I != 3; I++)
 598    {
 599       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 600          return false;
 601    }
 602
 603    return true;
 604 }
 605                                                                         /*}}}*/
 606 // AddStatusSize - Add the size of the status files                     /*{{{*/
 607 // ---------------------------------------------------------------------
 608 /* This adds the size of all the status files to the size counter */
 609 bool pkgAddStatusSize(unsigned long &TotalSize)
 610 {
 611    // Grab the file names
 612    string xstatus = _config->FindFile("Dir::State::xstatus");
 613    string userstatus = _config->FindFile("Dir::State::userstatus");
 614    string status = _config->FindFile("Dir::State::status");
 615
 616    // Grab the sizes
 617    struct stat Buf;
 618    if (stat(xstatus.c_str(),&Buf) == 0)
 619       TotalSize += Buf.st_size;
 620    if (stat(userstatus.c_str(),&Buf) == 0)
 621       TotalSize += Buf.st_size;
 622    if (stat(status.c_str(),&Buf) != 0)
 623       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 624    TotalSize += Buf.st_size;
 625
 626    return true;
 627 }
 628                                                                         /*}}}*/
 629 // MergeStatus - Add the status files to the cache                      /*{{{*/
 630 // ---------------------------------------------------------------------
 631 /* This adds the status files to the map */
 632 bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 633                     unsigned long &CurrentSize,unsigned long TotalSize)
 634 {
 635    // Grab the file names
 636    string Status[3];
 637    Status[0] = _config->FindFile("Dir::State::xstatus");
 638    Status[1]= _config->FindFile("Dir::State::userstatus");
 639    Status[2] = _config->FindFile("Dir::State::status");
 640
 641    for (int I = 0; I != 3; I++)
 642    {
 643       // Check if the file exists and it is not the primary status file.
 644       string File = Status[I];
 645       if (I != 2 && FileExists(File) == false)
 646          continue;
 647
 648       FileFd Pkg(File,FileFd::ReadOnly);
 649       debListParser Parser(Pkg);
 650       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 651       if (_error->PendingError() == true)
 652          return _error->Error("Problem opening %s",File.c_str());
 653       CurrentSize += Pkg.Size();
 654
 655       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 656       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 657          return _error->Error("Problem with SelectFile %s",File.c_str());
 658
 659       if (Gen.MergeList(Parser) == false)
 660          return _error->Error("Problem with MergeList %s",File.c_str());
 661       Progress.Progress(Pkg.Size());
 662    }
 663
 664    return true;
 665 }
 666                                                                         /*}}}*/
 667 // GenerateSrcCache - Write the source package lists to the map         /*{{{*/
 668 // ---------------------------------------------------------------------
 669 /* This puts the source package cache into the given generator. */
 670 bool pkgGenerateSrcCache(pkgSourceList &List,OpProgress &Progress,
 671                          pkgCacheGenerator &Gen,
 672                          unsigned long &CurrentSize,unsigned long &TotalSize)
 673 {
 674    string ListDir = _config->FindDir("Dir::State::lists");
 675
 676    // Prepare the progress indicator
 677    TotalSize = 0;
 678    struct stat Buf;
 679    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 680    {
 681       string File = ListDir + URItoFileName(I->PackagesURI());
 682       if (stat(File.c_str(),&Buf) != 0)
 683          continue;
 684       TotalSize += Buf.st_size;
 685    }
 686
 687    if (pkgAddStatusSize(TotalSize) == false)
 688       return false;
 689
 690    // Generate the pkg source cache
 691    CurrentSize = 0;
 692    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 693    {
 694       // Only cache deb source types.
 695       if (I->Type != pkgSourceList::Item::Deb)
 696          continue;
 697
 698       string File = ListDir + URItoFileName(I->PackagesURI());
 699
 700       if (FileExists(File) == false)
 701          continue;
 702
 703       FileFd Pkg(File,FileFd::ReadOnly);
 704       debListParser Parser(Pkg);
 705       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 706       if (_error->PendingError() == true)
 707          return _error->Error("Problem opening %s",File.c_str());
 708       CurrentSize += Pkg.Size();
 709
 710       Progress.SubProgress(0,I->PackagesInfo());
 711       if (Gen.SelectFile(File) == false)
 712          return _error->Error("Problem with SelectFile %s",File.c_str());
 713
 714       if (Gen.MergeList(Parser) == false)
 715          return _error->Error("Problem with MergeList %s",File.c_str());
 716
 717       // Check the release file
 718       string RFile = ListDir + URItoFileName(I->ReleaseURI());
 719       if (FileExists(RFile) == true)
 720       {
 721          FileFd Rel(RFile,FileFd::ReadOnly);
 722          if (_error->PendingError() == true)
 723             return false;
 724          Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 725       }
 726    }
 727
 728    return true;
 729 }
 730                                                                         /*}}}*/
 731 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 732 // ---------------------------------------------------------------------
 733 /* This copies the package source cache and then merges the status and
 734    xstatus files into it. */
 735 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 736 {
 737    unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
 738
 739    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 740
 741    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 742    bool SrcOk = pkgSrcCacheCheck(List);
 743    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 744
 745    // Rebuild the source and package caches
 746    if (SrcOk == false)
 747    {
 748       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 749       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 750
 751       /* Open the pkgcache, we want a new inode here so we do no corrupt
 752          existing mmaps */
 753       unlink(CacheFile.c_str());
 754       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 755       DynamicMMap Map(CacheF,MMap::Public,MapSize);
 756       if (_error->PendingError() == true)
 757          return false;
 758
 759       pkgCacheGenerator Gen(Map,Progress);
 760       unsigned long CurrentSize = 0;
 761       unsigned long TotalSize = 0;
 762       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 763          return false;
 764
 765       // Write the src cache
 766       Gen.GetCache().HeaderP->Dirty = false;
 767       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 768          return _error->Error("IO Error saving source cache");
 769       Gen.GetCache().HeaderP->Dirty = true;
 770
 771       // Merge in the source caches
 772       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 773    }
 774
 775    if (PkgOk == true)
 776    {
 777       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 778       return true;
 779    }
 780
 781    // We use the source cache to generate the package cache
 782    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 783    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 784
 785    /* Open the pkgcache, we want a new inode here so we do no corrupt
 786       existing mmaps */
 787    unlink(CacheFile.c_str());
 788    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 789    DynamicMMap Map(CacheF,MMap::Public,MapSize);
 790    if (_error->PendingError() == true)
 791       return false;
 792
 793    // Preload the map with the source cache
 794    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 795                     SCacheF.Size()) == false)
 796       return false;
 797
 798    pkgCacheGenerator Gen(Map,Progress);
 799
 800    // Compute the progress
 801    unsigned long TotalSize = 0;
 802    if (pkgAddStatusSize(TotalSize) == false)
 803       return false;
 804
 805    unsigned long CurrentSize = 0;
 806    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 807 }
 808                                                                         /*}}}*/
 809 // MakeStatusCacheMem - Returns a map for the status cache              /*{{{*/
 810 // ---------------------------------------------------------------------
 811 /* This creates a map object for the status cache. If the process has write
 812    access to the caches then it is the same as MakeStatusCache, otherwise it
 813    creates a memory block and puts the cache in there. */
 814 MMap *pkgMakeStatusCacheMem(pkgSourceList &List,OpProgress &Progress)
 815 {
 816    unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
 817
 818    /* If the cache file is writeable this is just a wrapper for
 819       MakeStatusCache */
 820    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 821    bool Writeable = (access(CacheFile.c_str(),W_OK) == 0) ||
 822                     (errno == ENOENT);
 823
 824    if (Writeable == true)
 825    {
 826       if (pkgMakeStatusCache(List,Progress) == false)
 827          return 0;
 828
 829       // Open the cache file
 830       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 831       if (_error->PendingError() == true)
 832          return 0;
 833
 834       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 835       if (_error->PendingError() == true)
 836       {
 837          delete Map;
 838          return 0;
 839       }
 840       return Map;
 841    }
 842
 843    // Mostly from MakeStatusCache..
 844    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 845
 846    bool SrcOk = pkgSrcCacheCheck(List);
 847    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 848
 849    // Rebuild the source and package caches
 850    if (SrcOk == false)
 851    {
 852       DynamicMMap *Map = new DynamicMMap(MMap::Public,MapSize);
 853       if (_error->PendingError() == true)
 854       {
 855          delete Map;
 856          return 0;
 857       }
 858
 859       pkgCacheGenerator Gen(*Map,Progress);
 860       unsigned long CurrentSize = 0;
 861       unsigned long TotalSize = 0;
 862       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 863       {
 864          delete Map;
 865          return 0;
 866       }
 867
 868       // Merge in the source caches
 869       if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 870       {
 871          delete Map;
 872          return 0;
 873       }
 874
 875       return Map;
 876    }
 877
 878    if (PkgOk == true)
 879    {
 880       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 881
 882       // Open the cache file
 883       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 884       if (_error->PendingError() == true)
 885          return 0;
 886
 887       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 888       if (_error->PendingError() == true)
 889       {
 890          delete Map;
 891          return 0;
 892       }
 893       return Map;
 894    }
 895
 896    // We use the source cache to generate the package cache
 897    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 898    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 899    DynamicMMap *Map = new DynamicMMap(MMap::Public,MapSize);
 900    if (_error->PendingError() == true)
 901    {
 902       delete Map;
 903       return 0;
 904    }
 905
 906    // Preload the map with the source cache
 907    if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 908                     SCacheF.Size()) == false)
 909    {
 910       delete Map;
 911       return 0;
 912    }
 913
 914    pkgCacheGenerator Gen(*Map,Progress);
 915
 916    // Compute the progress
 917    unsigned long TotalSize = 0;
 918    if (pkgAddStatusSize(TotalSize) == false)
 919    {
 920       delete Map;
 921       return 0;
 922    }
 923
 924    unsigned long CurrentSize = 0;
 925    if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 926    {
 927       delete Map;
 928       return 0;
 929    }
 930
 931    return Map;
 932 }
 933                                                                         /*}}}*/