apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24
  25 #include <apt-pkg/tagfile.h>
  26
  27 #include <apti18n.h>
  28
  29 #include <vector>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35 #include <system.h>
  36                                                                         /*}}}*/
  37 typedef vector<pkgIndexFile *>::iterator FileIterator;
  38
  39 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* We set the diry flag and make sure that is written to the disk */
  42 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  43                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  44                     FoundFileDeps(0)
  45 {
  46    CurrentFile = 0;
  47    memset(UniqHash,0,sizeof(UniqHash));
  48
  49    if (_error->PendingError() == true)
  50       return;
  51
  52    if (Map.Size() == 0)
  53    {
  54       // Setup the map interface..
  55       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  56       Map.RawAllocate(sizeof(pkgCache::Header));
  57       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  58
  59       // Starting header
  60       *Cache.HeaderP = pkgCache::Header();
  61       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  62       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  63       Cache.ReMap();
  64    }
  65    else
  66    {
  67       // Map directly from the existing file
  68       Cache.ReMap();
  69       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  70       if (Cache.VS != _system->VS)
  71       {
  72          _error->Error(_("Cache has an incompatible versioning system"));
  73          return;
  74       }
  75    }
  76
  77    Cache.HeaderP->Dirty = true;
  78    Map.Sync(0,sizeof(pkgCache::Header));
  79 }
  80                                                                         /*}}}*/
  81 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  82 // ---------------------------------------------------------------------
  83 /* We sync the data then unset the dirty flag in two steps so as to
  84    advoid a problem during a crash */
  85 pkgCacheGenerator::~pkgCacheGenerator()
  86 {
  87    if (_error->PendingError() == true)
  88       return;
  89    if (Map.Sync() == false)
  90       return;
  91
  92    Cache.HeaderP->Dirty = false;
  93    Map.Sync(0,sizeof(pkgCache::Header));
  94 }
  95                                                                         /*}}}*/
  96 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* This provides the generation of the entries in the cache. Each loop
  99    goes through a single package record from the underlying parse engine. */
 100 bool pkgCacheGenerator::MergeList(ListParser &List,
 101                                   pkgCache::VerIterator *OutVer)
 102 {
 103    List.Owner = this;
 104
 105    unsigned int Counter = 0;
 106    while (List.Step() == true)
 107    {
 108       // Get a pointer to the package structure
 109       string PackageName = List.Package();
 110       if (PackageName.empty() == true)
 111          return false;
 112
 113       pkgCache::PkgIterator Pkg;
 114       if (NewPackage(Pkg,PackageName) == false)
 115          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 116       Counter++;
 117       if (Counter % 100 == 0 && Progress != 0)
 118          Progress->Progress(List.Offset());
 119
 120       /* Get a pointer to the version structure. We know the list is sorted
 121          so we use that fact in the search. Insertion of new versions is
 122          done with correct sorting */
 123       string Version = List.Version();
 124       if (Version.empty() == true)
 125       {
 126          // we first process the package, then the descriptions
 127          // (this has the bonus that we get MMap error when we run out
 128          //  of MMap space)
 129          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 130             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 131                                  PackageName.c_str());
 132
 133          // Find the right version to write the description
 134          MD5SumValue CurMd5 = List.Description_md5();
 135          pkgCache::VerIterator Ver = Pkg.VersionList();
 136          map_ptrloc *LastVer = &Pkg->VersionList;
 137
 138          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 139          {
 140             pkgCache::DescIterator Desc = Ver.DescriptionList();
 141             map_ptrloc *LastDesc = &Ver->DescriptionList;
 142             bool duplicate=false;
 143
 144             // don't add a new description if we have one for the given
 145             // md5 && language
 146             for ( ; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++)
 147                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 148                    Desc.LanguageCode() == List.DescriptionLanguage())
 149                   duplicate=true;
 150             if(duplicate)
 151                continue;
 152
 153             for (Desc = Ver.DescriptionList();
 154                  Desc.end() == false;
 155                  LastDesc = &Desc->NextDesc, Desc++)
 156             {
 157                if (MD5SumValue(Desc.md5()) == CurMd5)
 158                {
 159                   // Add new description
 160                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 161                   Desc->ParentPkg = Pkg.Index();
 162
 163                   if (NewFileDesc(Desc,List) == false)
 164                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 165                   break;
 166                }
 167             }
 168          }
 169
 170          continue;
 171       }
 172
 173       pkgCache::VerIterator Ver = Pkg.VersionList();
 174       map_ptrloc *LastVer = &Pkg->VersionList;
 175       int Res = 1;
 176       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 177       {
 178          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 179          if (Res >= 0)
 180             break;
 181       }
 182
 183       /* We already have a version for this item, record that we
 184          saw it */
 185       unsigned long Hash = List.VersionHash();
 186       if (Res == 0 && Ver->Hash == Hash)
 187       {
 188          if (List.UsePackage(Pkg,Ver) == false)
 189             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 190                                  PackageName.c_str());
 191
 192          if (NewFileVer(Ver,List) == false)
 193             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 194                                  PackageName.c_str());
 195
 196          // Read only a single record and return
 197          if (OutVer != 0)
 198          {
 199             *OutVer = Ver;
 200             FoundFileDeps |= List.HasFileDeps();
 201             return true;
 202          }
 203
 204          continue;
 205       }
 206
 207       // Skip to the end of the same version set.
 208       if (Res == 0)
 209       {
 210          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 211          {
 212             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 213             if (Res != 0)
 214                break;
 215          }
 216       }
 217
 218       // Add a new version
 219       *LastVer = NewVersion(Ver,Version,*LastVer);
 220       Ver->ParentPkg = Pkg.Index();
 221       Ver->Hash = Hash;
 222
 223       if (List.NewVersion(Ver) == false)
 224          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 225                               PackageName.c_str());
 226
 227       if (List.UsePackage(Pkg,Ver) == false)
 228          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 229                               PackageName.c_str());
 230
 231       if (NewFileVer(Ver,List) == false)
 232          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 233                               PackageName.c_str());
 234
 235       // Read only a single record and return
 236       if (OutVer != 0)
 237       {
 238          *OutVer = Ver;
 239          FoundFileDeps |= List.HasFileDeps();
 240          return true;
 241       }
 242
 243       /* Record the Description data. Description data always exist in
 244          Packages and Translation-* files. */
 245       pkgCache::DescIterator Desc = Ver.DescriptionList();
 246       map_ptrloc *LastDesc = &Ver->DescriptionList;
 247
 248       // Skip to the end of description set
 249       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 250
 251       // Add new description
 252       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 253       Desc->ParentPkg = Pkg.Index();
 254
 255       if (NewFileDesc(Desc,List) == false)
 256          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 257    }
 258
 259    FoundFileDeps |= List.HasFileDeps();
 260
 261    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 262       return _error->Error(_("Wow, you exceeded the number of package "
 263                              "names this APT is capable of."));
 264    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 265       return _error->Error(_("Wow, you exceeded the number of versions "
 266                              "this APT is capable of."));
 267    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 268       return _error->Error(_("Wow, you exceeded the number of descriptions "
 269                              "this APT is capable of."));
 270    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 271       return _error->Error(_("Wow, you exceeded the number of dependencies "
 272                              "this APT is capable of."));
 273    return true;
 274 }
 275                                                                         /*}}}*/
 276 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 277 // ---------------------------------------------------------------------
 278 /* If we found any file depends while parsing the main list we need to
 279    resolve them. Since it is undesired to load the entire list of files
 280    into the cache as virtual packages we do a two stage effort. MergeList
 281    identifies the file depends and this creates Provdies for them by
 282    re-parsing all the indexs. */
 283 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 284 {
 285    List.Owner = this;
 286
 287    unsigned int Counter = 0;
 288    while (List.Step() == true)
 289    {
 290       string PackageName = List.Package();
 291       if (PackageName.empty() == true)
 292          return false;
 293       string Version = List.Version();
 294       if (Version.empty() == true)
 295          continue;
 296
 297       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 298       if (Pkg.end() == true)
 299          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 300                                 PackageName.c_str());
 301       Counter++;
 302       if (Counter % 100 == 0 && Progress != 0)
 303          Progress->Progress(List.Offset());
 304
 305       unsigned long Hash = List.VersionHash();
 306       pkgCache::VerIterator Ver = Pkg.VersionList();
 307       for (; Ver.end() == false; Ver++)
 308       {
 309          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 310          {
 311             if (List.CollectFileProvides(Cache,Ver) == false)
 312                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 313             break;
 314          }
 315       }
 316
 317       if (Ver.end() == true)
 318          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 319    }
 320
 321    return true;
 322 }
 323                                                                         /*}}}*/
 324 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 325 // ---------------------------------------------------------------------
 326 /* This creates a new package structure and adds it to the hash table */
 327 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 328 {
 329    Pkg = Cache.FindPkg(Name);
 330    if (Pkg.end() == false)
 331       return true;
 332
 333    // Get a structure
 334    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 335    if (Package == 0)
 336       return false;
 337
 338    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 339
 340    // Insert it into the hash table
 341    unsigned long Hash = Cache.Hash(Name);
 342    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 343    Cache.HeaderP->HashTable[Hash] = Package;
 344
 345    // Set the name and the ID
 346    Pkg->Name = Map.WriteString(Name);
 347    if (Pkg->Name == 0)
 348       return false;
 349    Pkg->ID = Cache.HeaderP->PackageCount++;
 350
 351    return true;
 352 }
 353                                                                         /*}}}*/
 354 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* */
 357 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 358                                    ListParser &List)
 359 {
 360    if (CurrentFile == 0)
 361       return true;
 362
 363    // Get a structure
 364    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 365    if (VerFile == 0)
 366       return 0;
 367
 368    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 369    VF->File = CurrentFile - Cache.PkgFileP;
 370
 371    // Link it to the end of the list
 372    map_ptrloc *Last = &Ver->FileList;
 373    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 374       Last = &V->NextFile;
 375    VF->NextFile = *Last;
 376    *Last = VF.Index();
 377
 378    VF->Offset = List.Offset();
 379    VF->Size = List.Size();
 380    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 381       Cache.HeaderP->MaxVerFileSize = VF->Size;
 382    Cache.HeaderP->VerFileCount++;
 383
 384    return true;
 385 }
 386                                                                         /*}}}*/
 387 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 388 // ---------------------------------------------------------------------
 389 /* This puts a version structure in the linked list */
 390 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 391                                             const string &VerStr,
 392                                             unsigned long Next)
 393 {
 394    // Get a structure
 395    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 396    if (Version == 0)
 397       return 0;
 398
 399    // Fill it in
 400    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 401    Ver->NextVer = Next;
 402    Ver->ID = Cache.HeaderP->VersionCount++;
 403    Ver->VerStr = Map.WriteString(VerStr);
 404    if (Ver->VerStr == 0)
 405       return 0;
 406
 407    return Version;
 408 }
 409                                                                         /*}}}*/
 410 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 411 // ---------------------------------------------------------------------
 412 /* */
 413 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 414                                    ListParser &List)
 415 {
 416    if (CurrentFile == 0)
 417       return true;
 418
 419    // Get a structure
 420    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 421    if (DescFile == 0)
 422       return 0;
 423
 424    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 425    DF->File = CurrentFile - Cache.PkgFileP;
 426
 427    // Link it to the end of the list
 428    map_ptrloc *Last = &Desc->FileList;
 429    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 430       Last = &D->NextFile;
 431
 432    DF->NextFile = *Last;
 433    *Last = DF.Index();
 434
 435    DF->Offset = List.Offset();
 436    DF->Size = List.Size();
 437    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 438       Cache.HeaderP->MaxDescFileSize = DF->Size;
 439    Cache.HeaderP->DescFileCount++;
 440
 441    return true;
 442 }
 443                                                                         /*}}}*/
 444 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 445 // ---------------------------------------------------------------------
 446 /* This puts a description structure in the linked list */
 447 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 448                                             const string &Lang,
 449                                             const MD5SumValue &md5sum,
 450                                             map_ptrloc Next)
 451 {
 452    // Get a structure
 453    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 454    if (Description == 0)
 455       return 0;
 456
 457    // Fill it in
 458    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 459    Desc->NextDesc = Next;
 460    Desc->ID = Cache.HeaderP->DescriptionCount++;
 461    Desc->language_code = Map.WriteString(Lang);
 462    Desc->md5sum = Map.WriteString(md5sum.Value());
 463
 464    return Description;
 465 }
 466                                                                         /*}}}*/
 467 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 468 // ---------------------------------------------------------------------
 469 /* This creates a dependency element in the tree. It is linked to the
 470    version and to the package that it is pointing to. */
 471 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 472                                                const string &PackageName,
 473                                                const string &Version,
 474                                                unsigned int Op,
 475                                                unsigned int Type)
 476 {
 477    pkgCache &Cache = Owner->Cache;
 478
 479    // Get a structure
 480    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 481    if (Dependency == 0)
 482       return false;
 483
 484    // Fill it in
 485    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 486    Dep->ParentVer = Ver.Index();
 487    Dep->Type = Type;
 488    Dep->CompareOp = Op;
 489    Dep->ID = Cache.HeaderP->DependsCount++;
 490
 491    // Locate the target package
 492    pkgCache::PkgIterator Pkg;
 493    if (Owner->NewPackage(Pkg,PackageName) == false)
 494       return false;
 495
 496    // Probe the reverse dependency list for a version string that matches
 497    if (Version.empty() == false)
 498    {
 499 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 500          if (I->Version != 0 && I.TargetVer() == Version)
 501             Dep->Version = I->Version;*/
 502       if (Dep->Version == 0)
 503          if ((Dep->Version = WriteString(Version)) == 0)
 504             return false;
 505    }
 506
 507    // Link it to the package
 508    Dep->Package = Pkg.Index();
 509    Dep->NextRevDepends = Pkg->RevDepends;
 510    Pkg->RevDepends = Dep.Index();
 511
 512    /* Link it to the version (at the end of the list)
 513       Caching the old end point speeds up generation substantially */
 514    if (OldDepVer != Ver)
 515    {
 516       OldDepLast = &Ver->DependsList;
 517       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 518          OldDepLast = &D->NextDepends;
 519       OldDepVer = Ver;
 520    }
 521
 522    // Is it a file dependency?
 523    if (PackageName[0] == '/')
 524       FoundFileDeps = true;
 525
 526    Dep->NextDepends = *OldDepLast;
 527    *OldDepLast = Dep.Index();
 528    OldDepLast = &Dep->NextDepends;
 529
 530    return true;
 531 }
 532                                                                         /*}}}*/
 533 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 534 // ---------------------------------------------------------------------
 535 /* */
 536 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 537                                                 const string &PackageName,
 538                                                 const string &Version)
 539 {
 540    pkgCache &Cache = Owner->Cache;
 541
 542    // We do not add self referencing provides
 543    if (Ver.ParentPkg().Name() == PackageName)
 544       return true;
 545
 546    // Get a structure
 547    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 548    if (Provides == 0)
 549       return false;
 550    Cache.HeaderP->ProvidesCount++;
 551
 552    // Fill it in
 553    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 554    Prv->Version = Ver.Index();
 555    Prv->NextPkgProv = Ver->ProvidesList;
 556    Ver->ProvidesList = Prv.Index();
 557    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 558       return false;
 559
 560    // Locate the target package
 561    pkgCache::PkgIterator Pkg;
 562    if (Owner->NewPackage(Pkg,PackageName) == false)
 563       return false;
 564
 565    // Link it to the package
 566    Prv->ParentPkg = Pkg.Index();
 567    Prv->NextProvides = Pkg->ProvidesList;
 568    Pkg->ProvidesList = Prv.Index();
 569
 570    return true;
 571 }
 572                                                                         /*}}}*/
 573 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 574 // ---------------------------------------------------------------------
 575 /* This is used to select which file is to be associated with all newly
 576    added versions. The caller is responsible for setting the IMS fields. */
 577 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 578                                    const pkgIndexFile &Index,
 579                                    unsigned long Flags)
 580 {
 581    // Get some space for the structure
 582    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 583    if (CurrentFile == Cache.PkgFileP)
 584       return false;
 585
 586    // Fill it in
 587    CurrentFile->FileName = Map.WriteString(File);
 588    CurrentFile->Site = WriteUniqString(Site);
 589    CurrentFile->NextFile = Cache.HeaderP->FileList;
 590    CurrentFile->Flags = Flags;
 591    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 592    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 593    PkgFileName = File;
 594    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 595    Cache.HeaderP->PackageFileCount++;
 596
 597    if (CurrentFile->FileName == 0)
 598       return false;
 599
 600    if (Progress != 0)
 601       Progress->SubProgress(Index.Size());
 602    return true;
 603 }
 604                                                                         /*}}}*/
 605 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 606 // ---------------------------------------------------------------------
 607 /* This is used to create handles to strings. Given the same text it
 608    always returns the same number */
 609 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 610                                                  unsigned int Size)
 611 {
 612    /* We use a very small transient hash table here, this speeds up generation
 613       by a fair amount on slower machines */
 614    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 615    if (Bucket != 0 &&
 616        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 617       return Bucket->String;
 618
 619    // Search for an insertion point
 620    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 621    int Res = 1;
 622    map_ptrloc *Last = &Cache.HeaderP->StringList;
 623    for (; I != Cache.StringItemP; Last = &I->NextItem,
 624         I = Cache.StringItemP + I->NextItem)
 625    {
 626       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 627       if (Res >= 0)
 628          break;
 629    }
 630
 631    // Match
 632    if (Res == 0)
 633    {
 634       Bucket = I;
 635       return I->String;
 636    }
 637
 638    // Get a structure
 639    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 640    if (Item == 0)
 641       return 0;
 642
 643    // Fill in the structure
 644    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 645    ItemP->NextItem = I - Cache.StringItemP;
 646    *Last = Item;
 647    ItemP->String = Map.WriteString(S,Size);
 648    if (ItemP->String == 0)
 649       return 0;
 650
 651    Bucket = ItemP;
 652    return ItemP->String;
 653 }
 654                                                                         /*}}}*/
 655
 656 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 657 // ---------------------------------------------------------------------
 658 /* This just verifies that each file in the list of index files exists,
 659    has matching attributes with the cache and the cache does not have
 660    any extra files. */
 661 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 662                           FileIterator End,MMap **OutMap = 0)
 663 {
 664    // No file, certainly invalid
 665    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 666       return false;
 667
 668    // Map it
 669    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 670    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 671    pkgCache Cache(Map);
 672    if (_error->PendingError() == true || Map->Size() == 0)
 673    {
 674       _error->Discard();
 675       return false;
 676    }
 677
 678    /* Now we check every index file, see if it is in the cache,
 679       verify the IMS data and check that it is on the disk too.. */
 680    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 681    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 682    for (; Start != End; Start++)
 683    {
 684       if ((*Start)->HasPackages() == false)
 685          continue;
 686
 687       if ((*Start)->Exists() == false)
 688       {
 689 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 690          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 691                           (*Start)->Describe().c_str());
 692 #endif
 693          continue;
 694       }
 695
 696       // FindInCache is also expected to do an IMS check.
 697       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 698       if (File.end() == true)
 699          return false;
 700
 701       Visited[File->ID] = true;
 702    }
 703
 704    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 705       if (Visited[I] == false)
 706          return false;
 707
 708    if (_error->PendingError() == true)
 709    {
 710       _error->Discard();
 711       return false;
 712    }
 713
 714    if (OutMap != 0)
 715       *OutMap = Map.UnGuard();
 716    return true;
 717 }
 718                                                                         /*}}}*/
 719 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 720 // ---------------------------------------------------------------------
 721 /* Size is kind of an abstract notion that is only used for the progress
 722    meter */
 723 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 724 {
 725    unsigned long TotalSize = 0;
 726    for (; Start != End; Start++)
 727    {
 728       if ((*Start)->HasPackages() == false)
 729          continue;
 730       TotalSize += (*Start)->Size();
 731    }
 732    return TotalSize;
 733 }
 734                                                                         /*}}}*/
 735 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 736 // ---------------------------------------------------------------------
 737 /* */
 738 static bool BuildCache(pkgCacheGenerator &Gen,
 739                        OpProgress &Progress,
 740                        unsigned long &CurrentSize,unsigned long TotalSize,
 741                        FileIterator Start, FileIterator End)
 742 {
 743    FileIterator I;
 744    for (I = Start; I != End; I++)
 745    {
 746       if ((*I)->HasPackages() == false)
 747          continue;
 748
 749       if ((*I)->Exists() == false)
 750          continue;
 751
 752       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 753       {
 754          _error->Warning("Duplicate sources.list entry %s",
 755                          (*I)->Describe().c_str());
 756          continue;
 757       }
 758
 759       unsigned long Size = (*I)->Size();
 760       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 761       CurrentSize += Size;
 762
 763       if ((*I)->Merge(Gen,Progress) == false)
 764          return false;
 765    }
 766
 767    if (Gen.HasFileDeps() == true)
 768    {
 769       Progress.Done();
 770       TotalSize = ComputeSize(Start, End);
 771       CurrentSize = 0;
 772       for (I = Start; I != End; I++)
 773       {
 774          unsigned long Size = (*I)->Size();
 775          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 776          CurrentSize += Size;
 777          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 778             return false;
 779       }
 780    }
 781
 782    return true;
 783 }
 784                                                                         /*}}}*/
 785 // MakeStatusCache - Construct the status cache                         /*{{{*/
 786 // ---------------------------------------------------------------------
 787 /* This makes sure that the status cache (the cache that has all
 788    index files from the sources list and all local ones) is ready
 789    to be mmaped. If OutMap is not zero then a MMap object representing
 790    the cache will be stored there. This is pretty much mandetory if you
 791    are using AllowMem. AllowMem lets the function be run as non-root
 792    where it builds the cache 'fast' into a memory buffer. */
 793 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 794                         MMap **OutMap,bool AllowMem)
 795 {
 796    unsigned long MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 797
 798    vector<pkgIndexFile *> Files;
 799    for (vector<metaIndex *>::const_iterator i = List.begin();
 800         i != List.end();
 801         i++)
 802    {
 803       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 804       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 805            j != Indexes->end();
 806            j++)
 807          Files.push_back (*j);
 808    }
 809
 810    unsigned long EndOfSource = Files.size();
 811    if (_system->AddStatusFiles(Files) == false)
 812       return false;
 813
 814    // Decide if we can write to the files..
 815    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 816    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 817
 818    // Decide if we can write to the cache
 819    bool Writeable = false;
 820    if (CacheFile.empty() == false)
 821       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 822    else
 823       if (SrcCacheFile.empty() == false)
 824          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 825
 826    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 827       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 828
 829    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 830
 831    // Cache is OK, Fin.
 832    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 833    {
 834       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 835       return true;
 836    }
 837
 838    /* At this point we know we need to reconstruct the package cache,
 839       begin. */
 840    SPtr<FileFd> CacheF;
 841    SPtr<DynamicMMap> Map;
 842    if (Writeable == true && CacheFile.empty() == false)
 843    {
 844       unlink(CacheFile.c_str());
 845       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 846       fchmod(CacheF->Fd(),0644);
 847       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 848       if (_error->PendingError() == true)
 849          return false;
 850    }
 851    else
 852    {
 853       // Just build it in memory..
 854       Map = new DynamicMMap(MMap::Public,MapSize);
 855    }
 856
 857    // Lets try the source cache.
 858    unsigned long CurrentSize = 0;
 859    unsigned long TotalSize = 0;
 860    if (CheckValidity(SrcCacheFile,Files.begin(),
 861                      Files.begin()+EndOfSource) == true)
 862    {
 863       // Preload the map with the source cache
 864       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 865       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 866                        SCacheF.Size()) == false)
 867          return false;
 868
 869       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 870
 871       // Build the status cache
 872       pkgCacheGenerator Gen(Map.Get(),&Progress);
 873       if (_error->PendingError() == true)
 874          return false;
 875       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 876                      Files.begin()+EndOfSource,Files.end()) == false)
 877          return false;
 878    }
 879    else
 880    {
 881       TotalSize = ComputeSize(Files.begin(),Files.end());
 882
 883       // Build the source cache
 884       pkgCacheGenerator Gen(Map.Get(),&Progress);
 885       if (_error->PendingError() == true)
 886          return false;
 887       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 888                      Files.begin(),Files.begin()+EndOfSource) == false)
 889          return false;
 890
 891       // Write it back
 892       if (Writeable == true && SrcCacheFile.empty() == false)
 893       {
 894          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 895          if (_error->PendingError() == true)
 896             return false;
 897
 898          fchmod(SCacheF.Fd(),0644);
 899
 900          // Write out the main data
 901          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 902             return _error->Error(_("IO Error saving source cache"));
 903          SCacheF.Sync();
 904
 905          // Write out the proper header
 906          Gen.GetCache().HeaderP->Dirty = false;
 907          if (SCacheF.Seek(0) == false ||
 908              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 909             return _error->Error(_("IO Error saving source cache"));
 910          Gen.GetCache().HeaderP->Dirty = true;
 911          SCacheF.Sync();
 912       }
 913
 914       // Build the status cache
 915       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 916                      Files.begin()+EndOfSource,Files.end()) == false)
 917          return false;
 918    }
 919
 920    if (_error->PendingError() == true)
 921       return false;
 922    if (OutMap != 0)
 923    {
 924       if (CacheF != 0)
 925       {
 926          delete Map.UnGuard();
 927          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 928       }
 929       else
 930       {
 931          *OutMap = Map.UnGuard();
 932       }
 933    }
 934
 935    return true;
 936 }
 937                                                                         /*}}}*/
 938 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 939 // ---------------------------------------------------------------------
 940 /* */
 941 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 942 {
 943    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
 944    vector<pkgIndexFile *> Files;
 945    unsigned long EndOfSource = Files.size();
 946    if (_system->AddStatusFiles(Files) == false)
 947       return false;
 948
 949    SPtr<DynamicMMap> Map;
 950    Map = new DynamicMMap(MMap::Public,MapSize);
 951    unsigned long CurrentSize = 0;
 952    unsigned long TotalSize = 0;
 953
 954    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 955
 956    // Build the status cache
 957    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 958    pkgCacheGenerator Gen(Map.Get(),&Progress);
 959    if (_error->PendingError() == true)
 960       return false;
 961    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 962                   Files.begin()+EndOfSource,Files.end()) == false)
 963       return false;
 964
 965    if (_error->PendingError() == true)
 966       return false;
 967    *OutMap = Map.UnGuard();
 968
 969    return true;
 970 }
 971                                                                         /*}}}*/