apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.38 1999/05/23 22:55:54 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25 #include <system.h>
  26
  27 #include <sys/stat.h>
  28 #include <unistd.h>
  29 #include <errno.h>
  30                                                                         /*}}}*/
  31
  32 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  33 // ---------------------------------------------------------------------
  34 /* We set the diry flag and make sure that is written to the disk */
  35 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  36                     Map(Map), Cache(Map), Progress(Prog)
  37 {
  38    if (_error->PendingError() == true)
  39       return;
  40
  41    if (Map.Size() == 0)
  42    {
  43       Map.RawAllocate(sizeof(pkgCache::Header));
  44       *Cache.HeaderP = pkgCache::Header();
  45    }
  46    Cache.HeaderP->Dirty = true;
  47    Map.Sync(0,sizeof(pkgCache::Header));
  48    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  49    memset(UniqHash,0,sizeof(UniqHash));
  50 }
  51                                                                         /*}}}*/
  52 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We sync the data then unset the dirty flag in two steps so as to
  55    advoid a problem during a crash */
  56 pkgCacheGenerator::~pkgCacheGenerator()
  57 {
  58    if (_error->PendingError() == true)
  59       return;
  60    if (Map.Sync() == false)
  61       return;
  62
  63    Cache.HeaderP->Dirty = false;
  64    Map.Sync(0,sizeof(pkgCache::Header));
  65 }
  66                                                                         /*}}}*/
  67 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* This provides the generation of the entries in the cache. Each loop
  70    goes through a single package record from the underlying parse engine. */
  71 bool pkgCacheGenerator::MergeList(ListParser &List)
  72 {
  73    List.Owner = this;
  74
  75    unsigned int Counter = 0;
  76    while (List.Step() == true)
  77    {
  78       // Get a pointer to the package structure
  79       string PackageName = List.Package();
  80       if (PackageName.empty() == true)
  81          return false;
  82
  83       pkgCache::PkgIterator Pkg;
  84       if (NewPackage(Pkg,PackageName) == false)
  85          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  86       Counter++;
  87       if (Counter % 100 == 0)
  88          Progress.Progress(List.Offset());
  89
  90       /* Get a pointer to the version structure. We know the list is sorted
  91          so we use that fact in the search. Insertion of new versions is
  92          done with correct sorting */
  93       string Version = List.Version();
  94       if (Version.empty() == true)
  95       {
  96          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
  97             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
  98          continue;
  99       }
 100
 101       pkgCache::VerIterator Ver = Pkg.VersionList();
 102       __apt_ptrloc *Last = &Pkg->VersionList;
 103       int Res = 1;
 104       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 105       {
 106          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 107                                  Ver.VerStr() + strlen(Ver.VerStr()));
 108          if (Res >= 0)
 109             break;
 110       }
 111
 112       /* We already have a version for this item, record that we
 113          saw it */
 114       unsigned long Hash = List.VersionHash();
 115       if (Res == 0 && Ver->Hash == Hash)
 116       {
 117          if (List.UsePackage(Pkg,Ver) == false)
 118             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 119
 120          if (NewFileVer(Ver,List) == false)
 121             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 122
 123          continue;
 124       }
 125
 126       // Skip to the end of the same version set.
 127       if (Res == 0)
 128       {
 129          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 130          {
 131             Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 132                                     Ver.VerStr() + strlen(Ver.VerStr()));
 133             if (Res != 0)
 134                break;
 135          }
 136       }
 137
 138       // Add a new version
 139       *Last = NewVersion(Ver,Version,*Last);
 140       Ver->ParentPkg = Pkg.Index();
 141       Ver->Hash = Hash;
 142       if (List.NewVersion(Ver) == false)
 143          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 144
 145       if (List.UsePackage(Pkg,Ver) == false)
 146          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 147
 148       if (NewFileVer(Ver,List) == false)
 149          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 150    }
 151
 152    return true;
 153 }
 154                                                                         /*}}}*/
 155 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This creates a new package structure and adds it to the hash table */
 158 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 159 {
 160    Pkg = Cache.FindPkg(Name);
 161    if (Pkg.end() == false)
 162       return true;
 163
 164    // Get a structure
 165    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 166    if (Package == 0)
 167       return false;
 168
 169    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 170
 171    // Insert it into the hash table
 172    unsigned long Hash = Cache.Hash(Name);
 173    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 174    Cache.HeaderP->HashTable[Hash] = Package;
 175
 176    // Set the name and the ID
 177    Pkg->Name = Map.WriteString(Name);
 178    if (Pkg->Name == 0)
 179       return false;
 180    Pkg->ID = Cache.HeaderP->PackageCount++;
 181
 182    return true;
 183 }
 184                                                                         /*}}}*/
 185 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 186 // ---------------------------------------------------------------------
 187 /* */
 188 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 189                                    ListParser &List)
 190 {
 191    // Get a structure
 192    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 193    if (VerFile == 0)
 194       return 0;
 195
 196    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 197    VF->File = CurrentFile - Cache.PkgFileP;
 198
 199    // Link it to the end of the list
 200    __apt_ptrloc *Last = &Ver->FileList;
 201    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 202       Last = &V->NextFile;
 203    VF->NextFile = *Last;
 204    *Last = VF.Index();
 205
 206    VF->Offset = List.Offset();
 207    VF->Size = List.Size();
 208    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 209       Cache.HeaderP->MaxVerFileSize = VF->Size;
 210    Cache.HeaderP->VerFileCount++;
 211
 212    return true;
 213 }
 214                                                                         /*}}}*/
 215 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 216 // ---------------------------------------------------------------------
 217 /* This puts a version structure in the linked list */
 218 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 219                                             string VerStr,
 220                                             unsigned long Next)
 221 {
 222    // Get a structure
 223    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 224    if (Version == 0)
 225       return 0;
 226
 227    // Fill it in
 228    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 229    Ver->NextVer = Next;
 230    Ver->ID = Cache.HeaderP->VersionCount++;
 231    Ver->VerStr = Map.WriteString(VerStr);
 232    if (Ver->VerStr == 0)
 233       return 0;
 234
 235    return Version;
 236 }
 237                                                                         /*}}}*/
 238 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 239 // ---------------------------------------------------------------------
 240 /* This creates a dependency element in the tree. It is linked to the
 241    version and to the package that it is pointing to. */
 242 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 243                                                string PackageName,
 244                                                string Version,
 245                                                unsigned int Op,
 246                                                unsigned int Type)
 247 {
 248    pkgCache &Cache = Owner->Cache;
 249
 250    // Get a structure
 251    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 252    if (Dependency == 0)
 253       return false;
 254
 255    // Fill it in
 256    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 257    Dep->ParentVer = Ver.Index();
 258    Dep->Type = Type;
 259    Dep->CompareOp = Op;
 260    Dep->ID = Cache.HeaderP->DependsCount++;
 261
 262    // Locate the target package
 263    pkgCache::PkgIterator Pkg;
 264    if (Owner->NewPackage(Pkg,PackageName) == false)
 265       return false;
 266
 267    // Probe the reverse dependency list for a version string that matches
 268    if (Version.empty() == false)
 269    {
 270 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++, Hit++)
 271          if (I->Version != 0 && I.TargetVer() == Version)
 272             Dep->Version = I->Version;*/
 273       if (Dep->Version == 0)
 274          if ((Dep->Version = WriteString(Version)) == 0)
 275             return false;
 276    }
 277
 278    // Link it to the package
 279    Dep->Package = Pkg.Index();
 280    Dep->NextRevDepends = Pkg->RevDepends;
 281    Pkg->RevDepends = Dep.Index();
 282
 283    /* Link it to the version (at the end of the list)
 284       Caching the old end point speeds up generation substantially */
 285    if (OldDepVer != Ver)
 286    {
 287       OldDepLast = &Ver->DependsList;
 288       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 289          OldDepLast = &D->NextDepends;
 290       OldDepVer = Ver;
 291    }
 292
 293    Dep->NextDepends = *OldDepLast;
 294    *OldDepLast = Dep.Index();
 295    OldDepLast = &Dep->NextDepends;
 296
 297    return true;
 298 }
 299                                                                         /*}}}*/
 300 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 301 // ---------------------------------------------------------------------
 302 /* */
 303 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 304                                                 string PackageName,
 305                                                 string Version)
 306 {
 307    pkgCache &Cache = Owner->Cache;
 308
 309    // We do not add self referencing provides
 310    if (Ver.ParentPkg().Name() == PackageName)
 311       return true;
 312
 313    // Get a structure
 314    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 315    if (Provides == 0)
 316       return false;
 317    Cache.HeaderP->ProvidesCount++;
 318
 319    // Fill it in
 320    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 321    Prv->Version = Ver.Index();
 322    Prv->NextPkgProv = Ver->ProvidesList;
 323    Ver->ProvidesList = Prv.Index();
 324    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 325       return false;
 326
 327    // Locate the target package
 328    pkgCache::PkgIterator Pkg;
 329    if (Owner->NewPackage(Pkg,PackageName) == false)
 330       return false;
 331
 332    // Link it to the package
 333    Prv->ParentPkg = Pkg.Index();
 334    Prv->NextProvides = Pkg->ProvidesList;
 335    Pkg->ProvidesList = Prv.Index();
 336
 337    return true;
 338 }
 339                                                                         /*}}}*/
 340 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* This is used to select which file is to be associated with all newly
 343    added versions. */
 344 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 345 {
 346    struct stat Buf;
 347    if (stat(File.c_str(),&Buf) == -1)
 348       return _error->Errno("stat","Couldn't stat ",File.c_str());
 349
 350    // Get some space for the structure
 351    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 352    if (CurrentFile == Cache.PkgFileP)
 353       return false;
 354
 355    // Fill it in
 356    CurrentFile->FileName = Map.WriteString(File);
 357    CurrentFile->Size = Buf.st_size;
 358    CurrentFile->mtime = Buf.st_mtime;
 359    CurrentFile->NextFile = Cache.HeaderP->FileList;
 360    CurrentFile->Flags = Flags;
 361    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 362    PkgFileName = File;
 363    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 364    Cache.HeaderP->PackageFileCount++;
 365
 366    if (CurrentFile->FileName == 0)
 367       return false;
 368
 369    Progress.SubProgress(Buf.st_size);
 370    return true;
 371 }
 372                                                                         /*}}}*/
 373 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* This is used to create handles to strings. Given the same text it
 376    always returns the same number */
 377 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 378                                                  unsigned int Size)
 379 {
 380    /* We use a very small transient hash table here, this speeds up generation
 381       by a fair amount on slower machines */
 382    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 383    if (Bucket != 0 &&
 384        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 385       return Bucket->String;
 386
 387    // Search for an insertion point
 388    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 389    int Res = 1;
 390    __apt_ptrloc *Last = &Cache.HeaderP->StringList;
 391    for (; I != Cache.StringItemP; Last = &I->NextItem,
 392         I = Cache.StringItemP + I->NextItem)
 393    {
 394       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 395       if (Res >= 0)
 396          break;
 397    }
 398
 399    // Match
 400    if (Res == 0)
 401    {
 402       Bucket = I;
 403       return I->String;
 404    }
 405
 406    // Get a structure
 407    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 408    if (Item == 0)
 409       return 0;
 410
 411    // Fill in the structure
 412    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 413    ItemP->NextItem = I - Cache.StringItemP;
 414    *Last = Item;
 415    ItemP->String = Map.WriteString(S,Size);
 416    if (ItemP->String == 0)
 417       return 0;
 418
 419    Bucket = ItemP;
 420    return ItemP->String;
 421 }
 422                                                                         /*}}}*/
 423
 424 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 425 // ---------------------------------------------------------------------
 426 /* The source cache is checked against the source list and the files
 427    on disk, any difference results in a false. */
 428 bool pkgSrcCacheCheck(pkgSourceList &List)
 429 {
 430    if (_error->PendingError() == true)
 431       return false;
 432
 433    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 434    string ListDir = _config->FindDir("Dir::State::lists");
 435
 436    // Count the number of missing files
 437    int Missing = 0;
 438    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 439    {
 440       // Only cache deb source types.
 441       if (I->Type != pkgSourceList::Item::Deb)
 442       {
 443          Missing++;
 444          continue;
 445       }
 446
 447       string File = ListDir + URItoFileName(I->PackagesURI());
 448       struct stat Buf;
 449       if (stat(File.c_str(),&Buf) != 0)
 450       {
 451          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 452                           I->PackagesInfo().c_str(),File.c_str());
 453          Missing++;
 454       }
 455    }
 456
 457    // Open the source package cache
 458    if (FileExists(CacheFile) == false)
 459       return false;
 460
 461    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 462    if (_error->PendingError() == true)
 463    {
 464       _error->Discard();
 465       return false;
 466    }
 467
 468    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 469    if (_error->PendingError() == true || Map.Size() == 0)
 470    {
 471       _error->Discard();
 472       return false;
 473    }
 474
 475    pkgCache Cache(Map);
 476    if (_error->PendingError() == true)
 477    {
 478       _error->Discard();
 479       return false;
 480    }
 481
 482    // They are certianly out of sync
 483    if (Cache.Head().PackageFileCount != List.size() - Missing)
 484       return false;
 485
 486    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 487    {
 488       // Search for a match in the source list
 489       bool Bad = true;
 490       for (pkgSourceList::const_iterator I = List.begin();
 491            I != List.end(); I++)
 492       {
 493          // Only cache deb source types.
 494          if (I->Type != pkgSourceList::Item::Deb)
 495             continue;
 496
 497          string File = ListDir + URItoFileName(I->PackagesURI());
 498          if (F.FileName() == File)
 499          {
 500             Bad = false;
 501             break;
 502          }
 503       }
 504
 505       // Check if the file matches what was cached
 506       Bad |= !F.IsOk();
 507       if (Bad == true)
 508          return false;
 509    }
 510
 511    return true;
 512 }
 513                                                                         /*}}}*/
 514 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 515 // ---------------------------------------------------------------------
 516 /* This does a simple check of all files used to compose the cache */
 517 bool pkgPkgCacheCheck(string CacheFile)
 518 {
 519    if (_error->PendingError() == true)
 520       return false;
 521
 522    // Open the source package cache
 523    if (FileExists(CacheFile) == false)
 524       return false;
 525
 526    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 527    if (_error->PendingError() == true)
 528    {
 529       _error->Discard();
 530       return false;
 531    }
 532
 533    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 534    if (_error->PendingError() == true || Map.Size() == 0)
 535    {
 536       _error->Discard();
 537       return false;
 538    }
 539
 540    pkgCache Cache(Map);
 541    if (_error->PendingError() == true)
 542    {
 543       _error->Discard();
 544       return false;
 545    }
 546
 547    // Status files that must be in the cache
 548    string Status[3];
 549    Status[0] = _config->FindFile("Dir::State::xstatus");
 550    Status[1]= _config->FindFile("Dir::State::userstatus");
 551    Status[2] = _config->FindFile("Dir::State::status");
 552
 553    // Cheack each file
 554    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 555    {
 556       if (F.IsOk() == false)
 557          return false;
 558
 559       // See if this is one of the status files
 560       for (int I = 0; I != 3; I++)
 561          if (F.FileName() == Status[I])
 562             Status[I] = string();
 563    }
 564
 565    // Make sure all the status files are loaded.
 566    for (int I = 0; I != 3; I++)
 567    {
 568       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 569          return false;
 570    }
 571
 572    return true;
 573 }
 574                                                                         /*}}}*/
 575 // AddSourcesSize - Add the size of the status files                    /*{{{*/
 576 // ---------------------------------------------------------------------
 577 /* This adds the size of all the status files to the size counter */
 578 static bool pkgAddSourcesSize(unsigned long &TotalSize)
 579 {
 580    // Grab the file names
 581    string xstatus = _config->FindFile("Dir::State::xstatus");
 582    string userstatus = _config->FindFile("Dir::State::userstatus");
 583    string status = _config->FindFile("Dir::State::status");
 584
 585    // Grab the sizes
 586    struct stat Buf;
 587    if (stat(xstatus.c_str(),&Buf) == 0)
 588       TotalSize += Buf.st_size;
 589    if (stat(userstatus.c_str(),&Buf) == 0)
 590       TotalSize += Buf.st_size;
 591    if (stat(status.c_str(),&Buf) != 0)
 592       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 593    TotalSize += Buf.st_size;
 594
 595    return true;
 596 }
 597                                                                         /*}}}*/
 598 // MergeStatus - Add the status files to the cache                      /*{{{*/
 599 // ---------------------------------------------------------------------
 600 /* This adds the status files to the map */
 601 static bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 602                            unsigned long &CurrentSize,unsigned long TotalSize)
 603 {
 604    // Grab the file names
 605    string Status[3];
 606    Status[0] = _config->FindFile("Dir::State::xstatus");
 607    Status[1]= _config->FindFile("Dir::State::userstatus");
 608    Status[2] = _config->FindFile("Dir::State::status");
 609
 610    for (int I = 0; I != 3; I++)
 611    {
 612       // Check if the file exists and it is not the primary status file.
 613       string File = Status[I];
 614       if (I != 2 && FileExists(File) == false)
 615          continue;
 616
 617       FileFd Pkg(File,FileFd::ReadOnly);
 618       debListParser Parser(Pkg);
 619       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 620       if (_error->PendingError() == true)
 621          return _error->Error("Problem opening %s",File.c_str());
 622       CurrentSize += Pkg.Size();
 623
 624       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 625       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 626          return _error->Error("Problem with SelectFile %s",File.c_str());
 627
 628       if (Gen.MergeList(Parser) == false)
 629          return _error->Error("Problem with MergeList %s",File.c_str());
 630       Progress.Progress(Pkg.Size());
 631    }
 632
 633    return true;
 634 }
 635                                                                         /*}}}*/
 636 // GenerateSrcCache - Write the source package lists to the map         /*{{{*/
 637 // ---------------------------------------------------------------------
 638 /* This puts the source package cache into the given generator. */
 639 bool pkgGenerateSrcCache(pkgSourceList &List,OpProgress &Progress,
 640                          pkgCacheGenerator &Gen,
 641                          unsigned long &CurrentSize,unsigned long &TotalSize)
 642 {
 643    string ListDir = _config->FindDir("Dir::State::lists");
 644
 645    // Prepare the progress indicator
 646    TotalSize = 0;
 647    struct stat Buf;
 648    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 649    {
 650       string File = ListDir + URItoFileName(I->PackagesURI());
 651       if (stat(File.c_str(),&Buf) != 0)
 652          continue;
 653       TotalSize += Buf.st_size;
 654    }
 655
 656    if (pkgAddSourcesSize(TotalSize) == false)
 657       return false;
 658
 659    // Generate the pkg source cache
 660    CurrentSize = 0;
 661    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 662    {
 663       // Only cache deb source types.
 664       if (I->Type != pkgSourceList::Item::Deb)
 665          continue;
 666
 667       string File = ListDir + URItoFileName(I->PackagesURI());
 668
 669       if (FileExists(File) == false)
 670          continue;
 671
 672       FileFd Pkg(File,FileFd::ReadOnly);
 673       debListParser Parser(Pkg);
 674       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 675       if (_error->PendingError() == true)
 676          return _error->Error("Problem opening %s",File.c_str());
 677       CurrentSize += Pkg.Size();
 678
 679       Progress.SubProgress(0,I->PackagesInfo());
 680       if (Gen.SelectFile(File) == false)
 681          return _error->Error("Problem with SelectFile %s",File.c_str());
 682
 683       if (Gen.MergeList(Parser) == false)
 684          return _error->Error("Problem with MergeList %s",File.c_str());
 685
 686       // Check the release file
 687       string RFile = ListDir + URItoFileName(I->ReleaseURI());
 688       if (FileExists(RFile) == true)
 689       {
 690          FileFd Rel(RFile,FileFd::ReadOnly);
 691          if (_error->PendingError() == true)
 692             return false;
 693          Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 694       }
 695    }
 696
 697    return true;
 698 }
 699                                                                         /*}}}*/
 700 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 701 // ---------------------------------------------------------------------
 702 /* This copies the package source cache and then merges the status and
 703    xstatus files into it. */
 704 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 705 {
 706    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 707
 708    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 709    bool SrcOk = pkgSrcCacheCheck(List);
 710    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 711
 712    // Rebuild the source and package caches
 713    if (SrcOk == false)
 714    {
 715       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 716       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 717       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 718       DynamicMMap Map(CacheF,MMap::Public);
 719       if (_error->PendingError() == true)
 720          return false;
 721
 722       pkgCacheGenerator Gen(Map,Progress);
 723       unsigned long CurrentSize = 0;
 724       unsigned long TotalSize = 0;
 725       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 726          return false;
 727
 728       // Write the src cache
 729       Gen.GetCache().HeaderP->Dirty = false;
 730       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 731          return _error->Error("IO Error saving source cache");
 732       Gen.GetCache().HeaderP->Dirty = true;
 733
 734       // Merge in the source caches
 735       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 736    }
 737
 738    if (PkgOk == true)
 739    {
 740       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 741       return true;
 742    }
 743
 744    // We use the source cache to generate the package cache
 745    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 746
 747    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 748    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 749    DynamicMMap Map(CacheF,MMap::Public);
 750    if (_error->PendingError() == true)
 751       return false;
 752
 753    // Preload the map with the source cache
 754    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 755                     SCacheF.Size()) == false)
 756       return false;
 757
 758    pkgCacheGenerator Gen(Map,Progress);
 759
 760    // Compute the progress
 761    unsigned long TotalSize = 0;
 762    if (pkgAddSourcesSize(TotalSize) == false)
 763       return false;
 764
 765    unsigned long CurrentSize = 0;
 766    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 767 }
 768                                                                         /*}}}*/
 769 // MakeStatusCacheMem - Returns a map for the status cache              /*{{{*/
 770 // ---------------------------------------------------------------------
 771 /* This creates a map object for the status cache. If the process has write
 772    access to the caches then it is the same as MakeStatusCache, otherwise it
 773    creates a memory block and puts the cache in there. */
 774 MMap *pkgMakeStatusCacheMem(pkgSourceList &List,OpProgress &Progress)
 775 {
 776    /* If the cache file is writeable this is just a wrapper for
 777       MakeStatusCache */
 778    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 779    bool Writeable = (access(CacheFile.c_str(),W_OK) == 0) ||
 780                     (errno == ENOENT);
 781
 782    if (Writeable == true)
 783    {
 784       if (pkgMakeStatusCache(List,Progress) == false)
 785          return 0;
 786
 787       // Open the cache file
 788       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 789       if (_error->PendingError() == true)
 790          return 0;
 791
 792       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 793       if (_error->PendingError() == true)
 794       {
 795          delete Map;
 796          return 0;
 797       }
 798       return Map;
 799    }
 800
 801    // Mostly from MakeStatusCache..
 802    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 803
 804    bool SrcOk = pkgSrcCacheCheck(List);
 805    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 806
 807    // Rebuild the source and package caches
 808    if (SrcOk == false)
 809    {
 810       DynamicMMap *Map = new DynamicMMap(MMap::Public);
 811       if (_error->PendingError() == true)
 812       {
 813          delete Map;
 814          return 0;
 815       }
 816
 817       pkgCacheGenerator Gen(*Map,Progress);
 818       unsigned long CurrentSize = 0;
 819       unsigned long TotalSize = 0;
 820       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 821       {
 822          delete Map;
 823          return 0;
 824       }
 825
 826       // Merge in the source caches
 827       if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 828       {
 829          delete Map;
 830          return 0;
 831       }
 832
 833       return Map;
 834    }
 835
 836    if (PkgOk == true)
 837    {
 838       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 839
 840       // Open the cache file
 841       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 842       if (_error->PendingError() == true)
 843          return 0;
 844
 845       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 846       if (_error->PendingError() == true)
 847       {
 848          delete Map;
 849          return 0;
 850       }
 851       return Map;
 852    }
 853
 854    // We use the source cache to generate the package cache
 855    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 856    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 857    DynamicMMap *Map = new DynamicMMap(MMap::Public);
 858    if (_error->PendingError() == true)
 859    {
 860       delete Map;
 861       return 0;
 862    }
 863
 864    // Preload the map with the source cache
 865    if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 866                     SCacheF.Size()) == false)
 867    {
 868       delete Map;
 869       return 0;
 870    }
 871
 872    pkgCacheGenerator Gen(*Map,Progress);
 873
 874    // Compute the progress
 875    unsigned long TotalSize = 0;
 876    if (pkgAddSourcesSize(TotalSize) == false)
 877    {
 878       delete Map;
 879       return 0;
 880    }
 881
 882    unsigned long CurrentSize = 0;
 883    if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 884    {
 885       delete Map;
 886       return 0;
 887    }
 888
 889    return Map;
 890 }
 891                                                                         /*}}}*/