apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apt-pkg/tagfile.h>
  30
  31 #include <apti18n.h>
  32
  33 #include <vector>
  34
  35 #include <sys/stat.h>
  36 #include <unistd.h>
  37 #include <errno.h>
  38 #include <stdio.h>
  39 #include <system.h>
  40                                                                         /*}}}*/
  41 typedef vector<pkgIndexFile *>::iterator FileIterator;
  42
  43 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* We set the diry flag and make sure that is written to the disk */
  46 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  47                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  48                     FoundFileDeps(0)
  49 {
  50    CurrentFile = 0;
  51    memset(UniqHash,0,sizeof(UniqHash));
  52
  53    if (_error->PendingError() == true)
  54       return;
  55
  56    if (Map.Size() == 0)
  57    {
  58       // Setup the map interface..
  59       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  60       Map.RawAllocate(sizeof(pkgCache::Header));
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  66       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  67       Cache.ReMap();
  68    }
  69    else
  70    {
  71       // Map directly from the existing file
  72       Cache.ReMap();
  73       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  74       if (Cache.VS != _system->VS)
  75       {
  76          _error->Error(_("Cache has an incompatible versioning system"));
  77          return;
  78       }
  79    }
  80
  81    Cache.HeaderP->Dirty = true;
  82    Map.Sync(0,sizeof(pkgCache::Header));
  83 }
  84                                                                         /*}}}*/
  85 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* We sync the data then unset the dirty flag in two steps so as to
  88    advoid a problem during a crash */
  89 pkgCacheGenerator::~pkgCacheGenerator()
  90 {
  91    if (_error->PendingError() == true)
  92       return;
  93    if (Map.Sync() == false)
  94       return;
  95
  96    Cache.HeaderP->Dirty = false;
  97    Map.Sync(0,sizeof(pkgCache::Header));
  98 }
  99                                                                         /*}}}*/
 100 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* This provides the generation of the entries in the cache. Each loop
 103    goes through a single package record from the underlying parse engine. */
 104 bool pkgCacheGenerator::MergeList(ListParser &List,
 105                                   pkgCache::VerIterator *OutVer)
 106 {
 107    List.Owner = this;
 108
 109    unsigned int Counter = 0;
 110    while (List.Step() == true)
 111    {
 112       // Get a pointer to the package structure
 113       string PackageName = List.Package();
 114       if (PackageName.empty() == true)
 115          return false;
 116
 117       pkgCache::PkgIterator Pkg;
 118       if (NewPackage(Pkg,PackageName) == false)
 119          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 120       Counter++;
 121       if (Counter % 100 == 0 && Progress != 0)
 122          Progress->Progress(List.Offset());
 123
 124       /* Get a pointer to the version structure. We know the list is sorted
 125          so we use that fact in the search. Insertion of new versions is
 126          done with correct sorting */
 127       string Version = List.Version();
 128       if (Version.empty() == true)
 129       {
 130          // we first process the package, then the descriptions
 131          // (this has the bonus that we get MMap error when we run out
 132          //  of MMap space)
 133          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 134             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 135                                  PackageName.c_str());
 136
 137          // Find the right version to write the description
 138          MD5SumValue CurMd5 = List.Description_md5();
 139          pkgCache::VerIterator Ver = Pkg.VersionList();
 140          map_ptrloc *LastVer = &Pkg->VersionList;
 141
 142          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 143          {
 144             pkgCache::DescIterator Desc = Ver.DescriptionList();
 145             map_ptrloc *LastDesc = &Ver->DescriptionList;
 146
 147             for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++)
 148             {
 149
 150                if (MD5SumValue(Desc.md5()) == CurMd5)
 151                {
 152                   // Add new description
 153                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 154                   Desc->ParentPkg = Pkg.Index();
 155
 156                   if (NewFileDesc(Desc,List) == false)
 157                      return _error->Error(_("Error occured while processing %s (NewFileDesc1)"),PackageName.c_str());
 158                   break;
 159                }
 160             }
 161          }
 162
 163          continue;
 164       }
 165
 166       pkgCache::VerIterator Ver = Pkg.VersionList();
 167       map_ptrloc *LastVer = &Pkg->VersionList;
 168       int Res = 1;
 169       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 170       {
 171          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 172          if (Res >= 0)
 173             break;
 174       }
 175
 176       /* We already have a version for this item, record that we
 177          saw it */
 178       unsigned long Hash = List.VersionHash();
 179       if (Res == 0 && Ver->Hash == Hash)
 180       {
 181          if (List.UsePackage(Pkg,Ver) == false)
 182             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 183                                  PackageName.c_str());
 184
 185          if (NewFileVer(Ver,List) == false)
 186             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 187                                  PackageName.c_str());
 188
 189          // Read only a single record and return
 190          if (OutVer != 0)
 191          {
 192             *OutVer = Ver;
 193             FoundFileDeps |= List.HasFileDeps();
 194             return true;
 195          }
 196
 197          continue;
 198       }
 199
 200       // Skip to the end of the same version set.
 201       if (Res == 0)
 202       {
 203          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 204          {
 205             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 206             if (Res != 0)
 207                break;
 208          }
 209       }
 210
 211       // Add a new version
 212       *LastVer = NewVersion(Ver,Version,*LastVer);
 213       Ver->ParentPkg = Pkg.Index();
 214       Ver->Hash = Hash;
 215
 216       if (List.NewVersion(Ver) == false)
 217          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 218                               PackageName.c_str());
 219
 220       if (List.UsePackage(Pkg,Ver) == false)
 221          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 222                               PackageName.c_str());
 223
 224       if (NewFileVer(Ver,List) == false)
 225          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 226                               PackageName.c_str());
 227
 228       // Read only a single record and return
 229       if (OutVer != 0)
 230       {
 231          *OutVer = Ver;
 232          FoundFileDeps |= List.HasFileDeps();
 233          return true;
 234       }
 235
 236       /* Record the Description data. Description data always exist in
 237          Packages and Translation-* files. */
 238       pkgCache::DescIterator Desc = Ver.DescriptionList();
 239       map_ptrloc *LastDesc = &Ver->DescriptionList;
 240
 241       // Skip to the end of description set
 242       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 243
 244       // Add new description
 245       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 246       Desc->ParentPkg = Pkg.Index();
 247
 248       if (NewFileDesc(Desc,List) == false)
 249          return _error->Error(_("Error occured while processing %s (NewFileDesc2)"),PackageName.c_str());
 250    }
 251
 252    FoundFileDeps |= List.HasFileDeps();
 253
 254    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 255       return _error->Error(_("Wow, you exceeded the number of package "
 256                              "names this APT is capable of."));
 257    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 258       return _error->Error(_("Wow, you exceeded the number of versions "
 259                              "this APT is capable of."));
 260    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 261       return _error->Error(_("Wow, you exceeded the number of descriptions "
 262                              "this APT is capable of."));
 263    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 264       return _error->Error(_("Wow, you exceeded the number of dependencies "
 265                              "this APT is capable of."));
 266    return true;
 267 }
 268                                                                         /*}}}*/
 269 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 270 // ---------------------------------------------------------------------
 271 /* If we found any file depends while parsing the main list we need to
 272    resolve them. Since it is undesired to load the entire list of files
 273    into the cache as virtual packages we do a two stage effort. MergeList
 274    identifies the file depends and this creates Provdies for them by
 275    re-parsing all the indexs. */
 276 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 277 {
 278    List.Owner = this;
 279
 280    unsigned int Counter = 0;
 281    while (List.Step() == true)
 282    {
 283       string PackageName = List.Package();
 284       if (PackageName.empty() == true)
 285          return false;
 286       string Version = List.Version();
 287       if (Version.empty() == true)
 288          continue;
 289
 290       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 291       if (Pkg.end() == true)
 292          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 293                                 PackageName.c_str());
 294       Counter++;
 295       if (Counter % 100 == 0 && Progress != 0)
 296          Progress->Progress(List.Offset());
 297
 298       unsigned long Hash = List.VersionHash();
 299       pkgCache::VerIterator Ver = Pkg.VersionList();
 300       for (; Ver.end() == false; Ver++)
 301       {
 302          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 303          {
 304             if (List.CollectFileProvides(Cache,Ver) == false)
 305                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 306             break;
 307          }
 308       }
 309
 310       if (Ver.end() == true)
 311          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 312    }
 313
 314    return true;
 315 }
 316                                                                         /*}}}*/
 317 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 318 // ---------------------------------------------------------------------
 319 /* This creates a new package structure and adds it to the hash table */
 320 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 321 {
 322    Pkg = Cache.FindPkg(Name);
 323    if (Pkg.end() == false)
 324       return true;
 325
 326    // Get a structure
 327    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 328    if (Package == 0)
 329       return false;
 330
 331    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 332
 333    // Insert it into the hash table
 334    unsigned long Hash = Cache.Hash(Name);
 335    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 336    Cache.HeaderP->HashTable[Hash] = Package;
 337
 338    // Set the name and the ID
 339    Pkg->Name = Map.WriteString(Name);
 340    if (Pkg->Name == 0)
 341       return false;
 342    Pkg->ID = Cache.HeaderP->PackageCount++;
 343
 344    return true;
 345 }
 346                                                                         /*}}}*/
 347 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 348 // ---------------------------------------------------------------------
 349 /* */
 350 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 351                                    ListParser &List)
 352 {
 353    if (CurrentFile == 0)
 354       return true;
 355
 356    // Get a structure
 357    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 358    if (VerFile == 0)
 359       return 0;
 360
 361    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 362    VF->File = CurrentFile - Cache.PkgFileP;
 363
 364    // Link it to the end of the list
 365    map_ptrloc *Last = &Ver->FileList;
 366    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 367       Last = &V->NextFile;
 368    VF->NextFile = *Last;
 369    *Last = VF.Index();
 370
 371    VF->Offset = List.Offset();
 372    VF->Size = List.Size();
 373    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 374       Cache.HeaderP->MaxVerFileSize = VF->Size;
 375    Cache.HeaderP->VerFileCount++;
 376
 377    return true;
 378 }
 379                                                                         /*}}}*/
 380 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* This puts a version structure in the linked list */
 383 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 384                                             const string &VerStr,
 385                                             unsigned long Next)
 386 {
 387    // Get a structure
 388    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 389    if (Version == 0)
 390       return 0;
 391
 392    // Fill it in
 393    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 394    Ver->NextVer = Next;
 395    Ver->ID = Cache.HeaderP->VersionCount++;
 396    Ver->VerStr = Map.WriteString(VerStr);
 397    if (Ver->VerStr == 0)
 398       return 0;
 399
 400    return Version;
 401 }
 402                                                                         /*}}}*/
 403 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 404 // ---------------------------------------------------------------------
 405 /* */
 406 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 407                                    ListParser &List)
 408 {
 409    if (CurrentFile == 0)
 410       return true;
 411
 412    // Get a structure
 413    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 414    if (DescFile == 0)
 415       return 0;
 416
 417    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 418    DF->File = CurrentFile - Cache.PkgFileP;
 419
 420    // Link it to the end of the list
 421    map_ptrloc *Last = &Desc->FileList;
 422    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 423       Last = &D->NextFile;
 424
 425    DF->NextFile = *Last;
 426    *Last = DF.Index();
 427
 428    DF->Offset = List.Offset();
 429    DF->Size = List.Size();
 430    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 431       Cache.HeaderP->MaxDescFileSize = DF->Size;
 432    Cache.HeaderP->DescFileCount++;
 433
 434    return true;
 435 }
 436                                                                         /*}}}*/
 437 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 438 // ---------------------------------------------------------------------
 439 /* This puts a description structure in the linked list */
 440 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 441                                             const string &Lang, const MD5SumValue &md5sum,
 442                                             map_ptrloc Next)
 443 {
 444    // Get a structure
 445    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 446    if (Description == 0)
 447       return 0;
 448
 449    // Fill it in
 450    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 451    Desc->NextDesc = Next;
 452    Desc->ID = Cache.HeaderP->DescriptionCount++;
 453    Desc->language_code = Map.WriteString(Lang);
 454    Desc->md5sum = Map.WriteString(md5sum.Value());
 455
 456    return Description;
 457 }
 458                                                                         /*}}}*/
 459 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 460 // ---------------------------------------------------------------------
 461 /* This creates a dependency element in the tree. It is linked to the
 462    version and to the package that it is pointing to. */
 463 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 464                                                const string &PackageName,
 465                                                const string &Version,
 466                                                unsigned int Op,
 467                                                unsigned int Type)
 468 {
 469    pkgCache &Cache = Owner->Cache;
 470
 471    // Get a structure
 472    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 473    if (Dependency == 0)
 474       return false;
 475
 476    // Fill it in
 477    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 478    Dep->ParentVer = Ver.Index();
 479    Dep->Type = Type;
 480    Dep->CompareOp = Op;
 481    Dep->ID = Cache.HeaderP->DependsCount++;
 482
 483    // Locate the target package
 484    pkgCache::PkgIterator Pkg;
 485    if (Owner->NewPackage(Pkg,PackageName) == false)
 486       return false;
 487
 488    // Probe the reverse dependency list for a version string that matches
 489    if (Version.empty() == false)
 490    {
 491 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 492          if (I->Version != 0 && I.TargetVer() == Version)
 493             Dep->Version = I->Version;*/
 494       if (Dep->Version == 0)
 495          if ((Dep->Version = WriteString(Version)) == 0)
 496             return false;
 497    }
 498
 499    // Link it to the package
 500    Dep->Package = Pkg.Index();
 501    Dep->NextRevDepends = Pkg->RevDepends;
 502    Pkg->RevDepends = Dep.Index();
 503
 504    /* Link it to the version (at the end of the list)
 505       Caching the old end point speeds up generation substantially */
 506    if (OldDepVer != Ver)
 507    {
 508       OldDepLast = &Ver->DependsList;
 509       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 510          OldDepLast = &D->NextDepends;
 511       OldDepVer = Ver;
 512    }
 513
 514    // Is it a file dependency?
 515    if (PackageName[0] == '/')
 516       FoundFileDeps = true;
 517
 518    Dep->NextDepends = *OldDepLast;
 519    *OldDepLast = Dep.Index();
 520    OldDepLast = &Dep->NextDepends;
 521
 522    return true;
 523 }
 524                                                                         /*}}}*/
 525 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 526 // ---------------------------------------------------------------------
 527 /* */
 528 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 529                                                 const string &PackageName,
 530                                                 const string &Version)
 531 {
 532    pkgCache &Cache = Owner->Cache;
 533
 534    // We do not add self referencing provides
 535    if (Ver.ParentPkg().Name() == PackageName)
 536       return true;
 537
 538    // Get a structure
 539    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 540    if (Provides == 0)
 541       return false;
 542    Cache.HeaderP->ProvidesCount++;
 543
 544    // Fill it in
 545    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 546    Prv->Version = Ver.Index();
 547    Prv->NextPkgProv = Ver->ProvidesList;
 548    Ver->ProvidesList = Prv.Index();
 549    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 550       return false;
 551
 552    // Locate the target package
 553    pkgCache::PkgIterator Pkg;
 554    if (Owner->NewPackage(Pkg,PackageName) == false)
 555       return false;
 556
 557    // Link it to the package
 558    Prv->ParentPkg = Pkg.Index();
 559    Prv->NextProvides = Pkg->ProvidesList;
 560    Pkg->ProvidesList = Prv.Index();
 561
 562    return true;
 563 }
 564                                                                         /*}}}*/
 565 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 566 // ---------------------------------------------------------------------
 567 /* This is used to select which file is to be associated with all newly
 568    added versions. The caller is responsible for setting the IMS fields. */
 569 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 570                                    const pkgIndexFile &Index,
 571                                    unsigned long Flags)
 572 {
 573    // Get some space for the structure
 574    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 575    if (CurrentFile == Cache.PkgFileP)
 576       return false;
 577
 578    // Fill it in
 579    CurrentFile->FileName = Map.WriteString(File);
 580    CurrentFile->Site = WriteUniqString(Site);
 581    CurrentFile->NextFile = Cache.HeaderP->FileList;
 582    CurrentFile->Flags = Flags;
 583    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 584    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 585    PkgFileName = File;
 586    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 587    Cache.HeaderP->PackageFileCount++;
 588
 589    if (CurrentFile->FileName == 0)
 590       return false;
 591
 592    if (Progress != 0)
 593       Progress->SubProgress(Index.Size());
 594    return true;
 595 }
 596                                                                         /*}}}*/
 597 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* This is used to create handles to strings. Given the same text it
 600    always returns the same number */
 601 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 602                                                  unsigned int Size)
 603 {
 604    /* We use a very small transient hash table here, this speeds up generation
 605       by a fair amount on slower machines */
 606    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 607    if (Bucket != 0 &&
 608        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 609       return Bucket->String;
 610
 611    // Search for an insertion point
 612    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 613    int Res = 1;
 614    map_ptrloc *Last = &Cache.HeaderP->StringList;
 615    for (; I != Cache.StringItemP; Last = &I->NextItem,
 616         I = Cache.StringItemP + I->NextItem)
 617    {
 618       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 619       if (Res >= 0)
 620          break;
 621    }
 622
 623    // Match
 624    if (Res == 0)
 625    {
 626       Bucket = I;
 627       return I->String;
 628    }
 629
 630    // Get a structure
 631    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 632    if (Item == 0)
 633       return 0;
 634
 635    // Fill in the structure
 636    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 637    ItemP->NextItem = I - Cache.StringItemP;
 638    *Last = Item;
 639    ItemP->String = Map.WriteString(S,Size);
 640    if (ItemP->String == 0)
 641       return 0;
 642
 643    Bucket = ItemP;
 644    return ItemP->String;
 645 }
 646                                                                         /*}}}*/
 647
 648 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 649 // ---------------------------------------------------------------------
 650 /* This just verifies that each file in the list of index files exists,
 651    has matching attributes with the cache and the cache does not have
 652    any extra files. */
 653 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 654                           FileIterator End,MMap **OutMap = 0)
 655 {
 656    // No file, certainly invalid
 657    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 658       return false;
 659
 660    // Map it
 661    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 662    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 663    pkgCache Cache(Map);
 664    if (_error->PendingError() == true || Map->Size() == 0)
 665    {
 666       _error->Discard();
 667       return false;
 668    }
 669
 670    /* Now we check every index file, see if it is in the cache,
 671       verify the IMS data and check that it is on the disk too.. */
 672    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 673    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 674    for (; Start != End; Start++)
 675    {
 676       if ((*Start)->HasPackages() == false)
 677          continue;
 678
 679       if ((*Start)->Exists() == false)
 680       {
 681 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 682          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 683                           (*Start)->Describe().c_str());
 684 #endif
 685          continue;
 686       }
 687
 688       // FindInCache is also expected to do an IMS check.
 689       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 690       if (File.end() == true)
 691          return false;
 692
 693       Visited[File->ID] = true;
 694    }
 695
 696    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 697       if (Visited[I] == false)
 698          return false;
 699
 700    if (_error->PendingError() == true)
 701    {
 702       _error->Discard();
 703       return false;
 704    }
 705
 706    if (OutMap != 0)
 707       *OutMap = Map.UnGuard();
 708    return true;
 709 }
 710                                                                         /*}}}*/
 711 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 712 // ---------------------------------------------------------------------
 713 /* Size is kind of an abstract notion that is only used for the progress
 714    meter */
 715 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 716 {
 717    unsigned long TotalSize = 0;
 718    for (; Start != End; Start++)
 719    {
 720       if ((*Start)->HasPackages() == false)
 721          continue;
 722       TotalSize += (*Start)->Size();
 723    }
 724    return TotalSize;
 725 }
 726                                                                         /*}}}*/
 727 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 728 // ---------------------------------------------------------------------
 729 /* */
 730 static bool BuildCache(pkgCacheGenerator &Gen,
 731                        OpProgress &Progress,
 732                        unsigned long &CurrentSize,unsigned long TotalSize,
 733                        FileIterator Start, FileIterator End)
 734 {
 735    FileIterator I;
 736    for (I = Start; I != End; I++)
 737    {
 738       if ((*I)->HasPackages() == false)
 739          continue;
 740
 741       if ((*I)->Exists() == false)
 742          continue;
 743
 744       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 745       {
 746          _error->Warning("Duplicate sources.list entry %s",
 747                          (*I)->Describe().c_str());
 748          continue;
 749       }
 750
 751       unsigned long Size = (*I)->Size();
 752       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 753       CurrentSize += Size;
 754
 755       if ((*I)->Merge(Gen,Progress) == false)
 756          return false;
 757    }
 758
 759    if (Gen.HasFileDeps() == true)
 760    {
 761       Progress.Done();
 762       TotalSize = ComputeSize(Start, End);
 763       CurrentSize = 0;
 764       for (I = Start; I != End; I++)
 765       {
 766          unsigned long Size = (*I)->Size();
 767          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 768          CurrentSize += Size;
 769          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 770             return false;
 771       }
 772    }
 773
 774    return true;
 775 }
 776                                                                         /*}}}*/
 777 // MakeStatusCache - Construct the status cache                         /*{{{*/
 778 // ---------------------------------------------------------------------
 779 /* This makes sure that the status cache (the cache that has all
 780    index files from the sources list and all local ones) is ready
 781    to be mmaped. If OutMap is not zero then a MMap object representing
 782    the cache will be stored there. This is pretty much mandetory if you
 783    are using AllowMem. AllowMem lets the function be run as non-root
 784    where it builds the cache 'fast' into a memory buffer. */
 785 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 786                         MMap **OutMap,bool AllowMem)
 787 {
 788    unsigned long MapSize = _config->FindI("APT::Cache-Limit",16*1024*1024);
 789
 790    vector<pkgIndexFile *> Files;
 791    for (vector<metaIndex *>::const_iterator i = List.begin();
 792         i != List.end();
 793         i++)
 794    {
 795       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 796       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 797            j != Indexes->end();
 798            j++)
 799          Files.push_back (*j);
 800    }
 801
 802    unsigned long EndOfSource = Files.size();
 803    if (_system->AddStatusFiles(Files) == false)
 804       return false;
 805
 806    // Decide if we can write to the files..
 807    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 808    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 809
 810    // Decide if we can write to the cache
 811    bool Writeable = false;
 812    if (CacheFile.empty() == false)
 813       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 814    else
 815       if (SrcCacheFile.empty() == false)
 816          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 817
 818    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 819       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 820
 821    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 822
 823    // Cache is OK, Fin.
 824    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 825    {
 826       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 827       return true;
 828    }
 829
 830    /* At this point we know we need to reconstruct the package cache,
 831       begin. */
 832    SPtr<FileFd> CacheF;
 833    SPtr<DynamicMMap> Map;
 834    if (Writeable == true && CacheFile.empty() == false)
 835    {
 836       unlink(CacheFile.c_str());
 837       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 838       fchmod(CacheF->Fd(),0644);
 839       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 840       if (_error->PendingError() == true)
 841          return false;
 842    }
 843    else
 844    {
 845       // Just build it in memory..
 846       Map = new DynamicMMap(MMap::Public,MapSize);
 847    }
 848
 849    // Lets try the source cache.
 850    unsigned long CurrentSize = 0;
 851    unsigned long TotalSize = 0;
 852    if (CheckValidity(SrcCacheFile,Files.begin(),
 853                      Files.begin()+EndOfSource) == true)
 854    {
 855       // Preload the map with the source cache
 856       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 857       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 858                        SCacheF.Size()) == false)
 859          return false;
 860
 861       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 862
 863       // Build the status cache
 864       pkgCacheGenerator Gen(Map.Get(),&Progress);
 865       if (_error->PendingError() == true)
 866          return false;
 867       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 868                      Files.begin()+EndOfSource,Files.end()) == false)
 869          return false;
 870    }
 871    else
 872    {
 873       TotalSize = ComputeSize(Files.begin(),Files.end());
 874
 875       // Build the source cache
 876       pkgCacheGenerator Gen(Map.Get(),&Progress);
 877       if (_error->PendingError() == true)
 878          return false;
 879       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 880                      Files.begin(),Files.begin()+EndOfSource) == false)
 881          return false;
 882
 883       // Write it back
 884       if (Writeable == true && SrcCacheFile.empty() == false)
 885       {
 886          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 887          if (_error->PendingError() == true)
 888             return false;
 889
 890          fchmod(SCacheF.Fd(),0644);
 891
 892          // Write out the main data
 893          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 894             return _error->Error(_("IO Error saving source cache"));
 895          SCacheF.Sync();
 896
 897          // Write out the proper header
 898          Gen.GetCache().HeaderP->Dirty = false;
 899          if (SCacheF.Seek(0) == false ||
 900              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 901             return _error->Error(_("IO Error saving source cache"));
 902          Gen.GetCache().HeaderP->Dirty = true;
 903          SCacheF.Sync();
 904       }
 905
 906       // Build the status cache
 907       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 908                      Files.begin()+EndOfSource,Files.end()) == false)
 909          return false;
 910    }
 911
 912    if (_error->PendingError() == true)
 913       return false;
 914    if (OutMap != 0)
 915    {
 916       if (CacheF != 0)
 917       {
 918          delete Map.UnGuard();
 919          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 920       }
 921       else
 922       {
 923          *OutMap = Map.UnGuard();
 924       }
 925    }
 926
 927    return true;
 928 }
 929                                                                         /*}}}*/
 930 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 931 // ---------------------------------------------------------------------
 932 /* */
 933 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 934 {
 935    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 936    vector<pkgIndexFile *> Files;
 937    unsigned long EndOfSource = Files.size();
 938    if (_system->AddStatusFiles(Files) == false)
 939       return false;
 940
 941    SPtr<DynamicMMap> Map;
 942    Map = new DynamicMMap(MMap::Public,MapSize);
 943    unsigned long CurrentSize = 0;
 944    unsigned long TotalSize = 0;
 945
 946    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 947
 948    // Build the status cache
 949    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 950    pkgCacheGenerator Gen(Map.Get(),&Progress);
 951    if (_error->PendingError() == true)
 952       return false;
 953    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 954                   Files.begin()+EndOfSource,Files.end()) == false)
 955       return false;
 956
 957    if (_error->PendingError() == true)
 958       return false;
 959    *OutMap = Map.UnGuard();
 960
 961    return true;
 962 }
 963                                                                         /*}}}*/