apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24
  25 #include <apt-pkg/tagfile.h>
  26
  27 #include <apti18n.h>
  28
  29 #include <vector>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35 #include <system.h>
  36                                                                         /*}}}*/
  37 typedef vector<pkgIndexFile *>::iterator FileIterator;
  38
  39 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* We set the diry flag and make sure that is written to the disk */
  42 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  43                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  44                     FoundFileDeps(0)
  45 {
  46    CurrentFile = 0;
  47    memset(UniqHash,0,sizeof(UniqHash));
  48
  49    if (_error->PendingError() == true)
  50       return;
  51
  52    if (Map.Size() == 0)
  53    {
  54       // Setup the map interface..
  55       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  56       Map.RawAllocate(sizeof(pkgCache::Header));
  57       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  58
  59       // Starting header
  60       *Cache.HeaderP = pkgCache::Header();
  61       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  62       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  63       Cache.ReMap();
  64    }
  65    else
  66    {
  67       // Map directly from the existing file
  68       Cache.ReMap();
  69       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  70       if (Cache.VS != _system->VS)
  71       {
  72          _error->Error(_("Cache has an incompatible versioning system"));
  73          return;
  74       }
  75    }
  76
  77    Cache.HeaderP->Dirty = true;
  78    Map.Sync(0,sizeof(pkgCache::Header));
  79 }
  80                                                                         /*}}}*/
  81 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  82 // ---------------------------------------------------------------------
  83 /* We sync the data then unset the dirty flag in two steps so as to
  84    advoid a problem during a crash */
  85 pkgCacheGenerator::~pkgCacheGenerator()
  86 {
  87    if (_error->PendingError() == true)
  88       return;
  89    if (Map.Sync() == false)
  90       return;
  91
  92    Cache.HeaderP->Dirty = false;
  93    Map.Sync(0,sizeof(pkgCache::Header));
  94 }
  95                                                                         /*}}}*/
  96 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* This provides the generation of the entries in the cache. Each loop
  99    goes through a single package record from the underlying parse engine. */
 100 bool pkgCacheGenerator::MergeList(ListParser &List,
 101                                   pkgCache::VerIterator *OutVer)
 102 {
 103    List.Owner = this;
 104
 105    unsigned int Counter = 0;
 106    while (List.Step() == true)
 107    {
 108       // Get a pointer to the package structure
 109       string PackageName = List.Package();
 110       if (PackageName.empty() == true)
 111          return false;
 112
 113       pkgCache::PkgIterator Pkg;
 114       if (NewPackage(Pkg,PackageName) == false)
 115          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 116       Counter++;
 117       if (Counter % 100 == 0 && Progress != 0)
 118          Progress->Progress(List.Offset());
 119
 120       /* Get a pointer to the version structure. We know the list is sorted
 121          so we use that fact in the search. Insertion of new versions is
 122          done with correct sorting */
 123       string Version = List.Version();
 124       if (Version.empty() == true)
 125       {
 126          // we first process the package, then the descriptions
 127          // (this has the bonus that we get MMap error when we run out
 128          //  of MMap space)
 129          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 130             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 131                                  PackageName.c_str());
 132
 133          // Find the right version to write the description
 134          MD5SumValue CurMd5 = List.Description_md5();
 135          pkgCache::VerIterator Ver = Pkg.VersionList();
 136          map_ptrloc *LastVer = &Pkg->VersionList;
 137
 138          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 139          {
 140             pkgCache::DescIterator Desc = Ver.DescriptionList();
 141             map_ptrloc *LastDesc = &Ver->DescriptionList;
 142             bool duplicate=false;
 143
 144             // don't add a new description if we have one for the given
 145             // md5 && language
 146             for ( ; Desc.end() == false; Desc++)
 147                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 148                    Desc.LanguageCode() == List.DescriptionLanguage())
 149                   duplicate=true;
 150             if(duplicate)
 151                continue;
 152
 153             for (Desc = Ver.DescriptionList();
 154                  Desc.end() == false;
 155                  LastDesc = &Desc->NextDesc, Desc++)
 156             {
 157                if (MD5SumValue(Desc.md5()) == CurMd5)
 158                {
 159                   // Add new description
 160                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 161                   Desc->ParentPkg = Pkg.Index();
 162
 163                   if (NewFileDesc(Desc,List) == false)
 164                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 165                   break;
 166                }
 167             }
 168          }
 169
 170          continue;
 171       }
 172
 173       pkgCache::VerIterator Ver = Pkg.VersionList();
 174       map_ptrloc *LastVer = &Pkg->VersionList;
 175       int Res = 1;
 176       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 177       {
 178          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 179          if (Res >= 0)
 180             break;
 181       }
 182
 183       /* We already have a version for this item, record that we
 184          saw it */
 185       unsigned long Hash = List.VersionHash();
 186       if (Res == 0 && Ver->Hash == Hash)
 187       {
 188          if (List.UsePackage(Pkg,Ver) == false)
 189             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 190                                  PackageName.c_str());
 191
 192          if (NewFileVer(Ver,List) == false)
 193             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 194                                  PackageName.c_str());
 195
 196          // Read only a single record and return
 197          if (OutVer != 0)
 198          {
 199             *OutVer = Ver;
 200             FoundFileDeps |= List.HasFileDeps();
 201             return true;
 202          }
 203
 204          continue;
 205       }
 206
 207       // Skip to the end of the same version set.
 208       if (Res == 0)
 209       {
 210          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 211          {
 212             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 213             if (Res != 0)
 214                break;
 215          }
 216       }
 217
 218       // Add a new version
 219       *LastVer = NewVersion(Ver,Version,*LastVer);
 220       Ver->ParentPkg = Pkg.Index();
 221       Ver->Hash = Hash;
 222
 223       if (List.NewVersion(Ver) == false)
 224          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 225                               PackageName.c_str());
 226
 227       if (List.UsePackage(Pkg,Ver) == false)
 228          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 229                               PackageName.c_str());
 230
 231       if (NewFileVer(Ver,List) == false)
 232          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 233                               PackageName.c_str());
 234
 235       // Read only a single record and return
 236       if (OutVer != 0)
 237       {
 238          *OutVer = Ver;
 239          FoundFileDeps |= List.HasFileDeps();
 240          return true;
 241       }
 242
 243       /* Record the Description data. Description data always exist in
 244          Packages and Translation-* files. */
 245       pkgCache::DescIterator Desc = Ver.DescriptionList();
 246       map_ptrloc *LastDesc = &Ver->DescriptionList;
 247
 248       // Skip to the end of description set
 249       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 250
 251       // Add new description
 252       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 253       Desc->ParentPkg = Pkg.Index();
 254
 255       if (NewFileDesc(Desc,List) == false)
 256          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 257    }
 258
 259    FoundFileDeps |= List.HasFileDeps();
 260
 261    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 262       return _error->Error(_("Wow, you exceeded the number of package "
 263                              "names this APT is capable of."));
 264    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 265       return _error->Error(_("Wow, you exceeded the number of versions "
 266                              "this APT is capable of."));
 267    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 268       return _error->Error(_("Wow, you exceeded the number of descriptions "
 269                              "this APT is capable of."));
 270    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 271       return _error->Error(_("Wow, you exceeded the number of dependencies "
 272                              "this APT is capable of."));
 273    return true;
 274 }
 275                                                                         /*}}}*/
 276 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 277 // ---------------------------------------------------------------------
 278 /* If we found any file depends while parsing the main list we need to
 279    resolve them. Since it is undesired to load the entire list of files
 280    into the cache as virtual packages we do a two stage effort. MergeList
 281    identifies the file depends and this creates Provdies for them by
 282    re-parsing all the indexs. */
 283 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 284 {
 285    List.Owner = this;
 286
 287    unsigned int Counter = 0;
 288    while (List.Step() == true)
 289    {
 290       string PackageName = List.Package();
 291       if (PackageName.empty() == true)
 292          return false;
 293       string Version = List.Version();
 294       if (Version.empty() == true)
 295          continue;
 296
 297       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 298       if (Pkg.end() == true)
 299          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 300                                 PackageName.c_str());
 301       Counter++;
 302       if (Counter % 100 == 0 && Progress != 0)
 303          Progress->Progress(List.Offset());
 304
 305       unsigned long Hash = List.VersionHash();
 306       pkgCache::VerIterator Ver = Pkg.VersionList();
 307       for (; Ver.end() == false; Ver++)
 308       {
 309          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 310          {
 311             if (List.CollectFileProvides(Cache,Ver) == false)
 312                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 313             break;
 314          }
 315       }
 316
 317       if (Ver.end() == true)
 318          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 319    }
 320
 321    return true;
 322 }
 323                                                                         /*}}}*/
 324 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 325 // ---------------------------------------------------------------------
 326 /* This creates a new package structure and adds it to the hash table */
 327 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 328 {
 329    Pkg = Cache.FindPkg(Name);
 330    if (Pkg.end() == false)
 331       return true;
 332
 333    // Get a structure
 334    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 335    if (Package == 0)
 336       return false;
 337
 338    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 339
 340    // Insert it into the hash table
 341    unsigned long Hash = Cache.Hash(Name);
 342    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 343    Cache.HeaderP->HashTable[Hash] = Package;
 344
 345    // Set the name and the ID
 346    Pkg->Name = Map.WriteString(Name);
 347    if (Pkg->Name == 0)
 348       return false;
 349    Pkg->ID = Cache.HeaderP->PackageCount++;
 350
 351    return true;
 352 }
 353                                                                         /*}}}*/
 354 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* */
 357 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 358                                    ListParser &List)
 359 {
 360    if (CurrentFile == 0)
 361       return true;
 362
 363    // Get a structure
 364    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 365    if (VerFile == 0)
 366       return 0;
 367
 368    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 369    VF->File = CurrentFile - Cache.PkgFileP;
 370
 371    // Link it to the end of the list
 372    map_ptrloc *Last = &Ver->FileList;
 373    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 374       Last = &V->NextFile;
 375    VF->NextFile = *Last;
 376    *Last = VF.Index();
 377
 378    VF->Offset = List.Offset();
 379    VF->Size = List.Size();
 380    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 381       Cache.HeaderP->MaxVerFileSize = VF->Size;
 382    Cache.HeaderP->VerFileCount++;
 383
 384    return true;
 385 }
 386                                                                         /*}}}*/
 387 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 388 // ---------------------------------------------------------------------
 389 /* This puts a version structure in the linked list */
 390 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 391                                             const string &VerStr,
 392                                             unsigned long Next)
 393 {
 394    // Get a structure
 395    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 396    if (Version == 0)
 397       return 0;
 398
 399    // Fill it in
 400    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 401    Ver->NextVer = Next;
 402    Ver->ID = Cache.HeaderP->VersionCount++;
 403    Ver->VerStr = Map.WriteString(VerStr);
 404    if (Ver->VerStr == 0)
 405       return 0;
 406
 407    return Version;
 408 }
 409                                                                         /*}}}*/
 410 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 411 // ---------------------------------------------------------------------
 412 /* */
 413 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 414                                    ListParser &List)
 415 {
 416    if (CurrentFile == 0)
 417       return true;
 418
 419    // Get a structure
 420    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 421    if (DescFile == 0)
 422       return 0;
 423
 424    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 425    DF->File = CurrentFile - Cache.PkgFileP;
 426
 427    // Link it to the end of the list
 428    map_ptrloc *Last = &Desc->FileList;
 429    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 430       Last = &D->NextFile;
 431
 432    DF->NextFile = *Last;
 433    *Last = DF.Index();
 434
 435    DF->Offset = List.Offset();
 436    DF->Size = List.Size();
 437    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 438       Cache.HeaderP->MaxDescFileSize = DF->Size;
 439    Cache.HeaderP->DescFileCount++;
 440
 441    return true;
 442 }
 443                                                                         /*}}}*/
 444 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 445 // ---------------------------------------------------------------------
 446 /* This puts a description structure in the linked list */
 447 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 448                                             const string &Lang,
 449                                             const MD5SumValue &md5sum,
 450                                             map_ptrloc Next)
 451 {
 452    // Get a structure
 453    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 454    if (Description == 0)
 455       return 0;
 456
 457    // Fill it in
 458    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 459    Desc->NextDesc = Next;
 460    Desc->ID = Cache.HeaderP->DescriptionCount++;
 461    Desc->language_code = Map.WriteString(Lang);
 462    Desc->md5sum = Map.WriteString(md5sum.Value());
 463
 464    return Description;
 465 }
 466                                                                         /*}}}*/
 467 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 468 // ---------------------------------------------------------------------
 469 /* This creates a dependency element in the tree. It is linked to the
 470    version and to the package that it is pointing to. */
 471 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 472                                                const string &PackageName,
 473                                                const string &Version,
 474                                                unsigned int Op,
 475                                                unsigned int Type)
 476 {
 477    pkgCache &Cache = Owner->Cache;
 478
 479    // Get a structure
 480    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 481    if (Dependency == 0)
 482       return false;
 483
 484    // Fill it in
 485    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 486    Dep->ParentVer = Ver.Index();
 487    Dep->Type = Type;
 488    Dep->CompareOp = Op;
 489    Dep->ID = Cache.HeaderP->DependsCount++;
 490
 491    // Locate the target package
 492    pkgCache::PkgIterator Pkg;
 493    if (Owner->NewPackage(Pkg,PackageName) == false)
 494       return false;
 495
 496    // Probe the reverse dependency list for a version string that matches
 497    if (Version.empty() == false)
 498    {
 499 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 500          if (I->Version != 0 && I.TargetVer() == Version)
 501             Dep->Version = I->Version;*/
 502       if (Dep->Version == 0)
 503          if ((Dep->Version = WriteString(Version)) == 0)
 504             return false;
 505    }
 506
 507    // Link it to the package
 508    Dep->Package = Pkg.Index();
 509    Dep->NextRevDepends = Pkg->RevDepends;
 510    Pkg->RevDepends = Dep.Index();
 511
 512    /* Link it to the version (at the end of the list)
 513       Caching the old end point speeds up generation substantially */
 514    if (OldDepVer != Ver)
 515    {
 516       OldDepLast = &Ver->DependsList;
 517       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 518          OldDepLast = &D->NextDepends;
 519       OldDepVer = Ver;
 520    }
 521
 522    // Is it a file dependency?
 523    if (PackageName[0] == '/')
 524       FoundFileDeps = true;
 525
 526    Dep->NextDepends = *OldDepLast;
 527    *OldDepLast = Dep.Index();
 528    OldDepLast = &Dep->NextDepends;
 529
 530    return true;
 531 }
 532                                                                         /*}}}*/
 533 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 534 // ---------------------------------------------------------------------
 535 /* */
 536 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 537                                                 const string &PackageName,
 538                                                 const string &Version)
 539 {
 540    pkgCache &Cache = Owner->Cache;
 541
 542    // We do not add self referencing provides
 543    if (Ver.ParentPkg().Name() == PackageName)
 544       return true;
 545
 546    // Get a structure
 547    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 548    if (Provides == 0)
 549       return false;
 550    Cache.HeaderP->ProvidesCount++;
 551
 552    // Fill it in
 553    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 554    Prv->Version = Ver.Index();
 555    Prv->NextPkgProv = Ver->ProvidesList;
 556    Ver->ProvidesList = Prv.Index();
 557    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 558       return false;
 559
 560    // Locate the target package
 561    pkgCache::PkgIterator Pkg;
 562    if (Owner->NewPackage(Pkg,PackageName) == false)
 563       return false;
 564
 565    // Link it to the package
 566    Prv->ParentPkg = Pkg.Index();
 567    Prv->NextProvides = Pkg->ProvidesList;
 568    Pkg->ProvidesList = Prv.Index();
 569
 570    return true;
 571 }
 572                                                                         /*}}}*/
 573 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 574 // ---------------------------------------------------------------------
 575 /* This is used to select which file is to be associated with all newly
 576    added versions. The caller is responsible for setting the IMS fields. */
 577 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 578                                    const pkgIndexFile &Index,
 579                                    unsigned long Flags)
 580 {
 581    // Get some space for the structure
 582    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 583    if (CurrentFile == Cache.PkgFileP)
 584       return false;
 585
 586    // Fill it in
 587    CurrentFile->FileName = Map.WriteString(File);
 588    CurrentFile->Site = WriteUniqString(Site);
 589    CurrentFile->NextFile = Cache.HeaderP->FileList;
 590    CurrentFile->Flags = Flags;
 591    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 592    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 593    PkgFileName = File;
 594    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 595    Cache.HeaderP->PackageFileCount++;
 596
 597    if (CurrentFile->FileName == 0)
 598       return false;
 599
 600    if (Progress != 0)
 601       Progress->SubProgress(Index.Size());
 602    return true;
 603 }
 604                                                                         /*}}}*/
 605 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 606 // ---------------------------------------------------------------------
 607 /* This is used to create handles to strings. Given the same text it
 608    always returns the same number */
 609 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 610                                                  unsigned int Size)
 611 {
 612    /* We use a very small transient hash table here, this speeds up generation
 613       by a fair amount on slower machines */
 614    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 615    if (Bucket != 0 &&
 616        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 617       return Bucket->String;
 618
 619    // Search for an insertion point
 620    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 621    int Res = 1;
 622    map_ptrloc *Last = &Cache.HeaderP->StringList;
 623    for (; I != Cache.StringItemP; Last = &I->NextItem,
 624         I = Cache.StringItemP + I->NextItem)
 625    {
 626       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 627       if (Res >= 0)
 628          break;
 629    }
 630
 631    // Match
 632    if (Res == 0)
 633    {
 634       Bucket = I;
 635       return I->String;
 636    }
 637
 638    // Get a structure
 639    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 640    if (Item == 0)
 641       return 0;
 642
 643    // Fill in the structure
 644    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 645    ItemP->NextItem = I - Cache.StringItemP;
 646    *Last = Item;
 647    ItemP->String = Map.WriteString(S,Size);
 648    if (ItemP->String == 0)
 649       return 0;
 650
 651    Bucket = ItemP;
 652    return ItemP->String;
 653 }
 654                                                                         /*}}}*/
 655 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 656 // ---------------------------------------------------------------------
 657 /* This just verifies that each file in the list of index files exists,
 658    has matching attributes with the cache and the cache does not have
 659    any extra files. */
 660 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 661                           FileIterator End,MMap **OutMap = 0)
 662 {
 663    // No file, certainly invalid
 664    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 665       return false;
 666
 667    // Map it
 668    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 669    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 670    pkgCache Cache(Map);
 671    if (_error->PendingError() == true || Map->Size() == 0)
 672    {
 673       _error->Discard();
 674       return false;
 675    }
 676
 677    /* Now we check every index file, see if it is in the cache,
 678       verify the IMS data and check that it is on the disk too.. */
 679    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 680    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 681    for (; Start != End; Start++)
 682    {
 683       if ((*Start)->HasPackages() == false)
 684          continue;
 685
 686       if ((*Start)->Exists() == false)
 687       {
 688 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 689          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 690                           (*Start)->Describe().c_str());
 691 #endif
 692          continue;
 693       }
 694
 695       // FindInCache is also expected to do an IMS check.
 696       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 697       if (File.end() == true)
 698          return false;
 699
 700       Visited[File->ID] = true;
 701    }
 702
 703    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 704       if (Visited[I] == false)
 705          return false;
 706
 707    if (_error->PendingError() == true)
 708    {
 709       _error->Discard();
 710       return false;
 711    }
 712
 713    if (OutMap != 0)
 714       *OutMap = Map.UnGuard();
 715    return true;
 716 }
 717                                                                         /*}}}*/
 718 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 719 // ---------------------------------------------------------------------
 720 /* Size is kind of an abstract notion that is only used for the progress
 721    meter */
 722 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 723 {
 724    unsigned long TotalSize = 0;
 725    for (; Start != End; Start++)
 726    {
 727       if ((*Start)->HasPackages() == false)
 728          continue;
 729       TotalSize += (*Start)->Size();
 730    }
 731    return TotalSize;
 732 }
 733                                                                         /*}}}*/
 734 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 735 // ---------------------------------------------------------------------
 736 /* */
 737 static bool BuildCache(pkgCacheGenerator &Gen,
 738                        OpProgress &Progress,
 739                        unsigned long &CurrentSize,unsigned long TotalSize,
 740                        FileIterator Start, FileIterator End)
 741 {
 742    FileIterator I;
 743    for (I = Start; I != End; I++)
 744    {
 745       if ((*I)->HasPackages() == false)
 746          continue;
 747
 748       if ((*I)->Exists() == false)
 749          continue;
 750
 751       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 752       {
 753          _error->Warning("Duplicate sources.list entry %s",
 754                          (*I)->Describe().c_str());
 755          continue;
 756       }
 757
 758       unsigned long Size = (*I)->Size();
 759       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 760       CurrentSize += Size;
 761
 762       if ((*I)->Merge(Gen,Progress) == false)
 763          return false;
 764    }
 765
 766    if (Gen.HasFileDeps() == true)
 767    {
 768       Progress.Done();
 769       TotalSize = ComputeSize(Start, End);
 770       CurrentSize = 0;
 771       for (I = Start; I != End; I++)
 772       {
 773          unsigned long Size = (*I)->Size();
 774          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 775          CurrentSize += Size;
 776          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 777             return false;
 778       }
 779    }
 780
 781    return true;
 782 }
 783                                                                         /*}}}*/
 784 // MakeStatusCache - Construct the status cache                         /*{{{*/
 785 // ---------------------------------------------------------------------
 786 /* This makes sure that the status cache (the cache that has all
 787    index files from the sources list and all local ones) is ready
 788    to be mmaped. If OutMap is not zero then a MMap object representing
 789    the cache will be stored there. This is pretty much mandetory if you
 790    are using AllowMem. AllowMem lets the function be run as non-root
 791    where it builds the cache 'fast' into a memory buffer. */
 792 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 793                         MMap **OutMap,bool AllowMem)
 794 {
 795    unsigned long MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 796
 797    vector<pkgIndexFile *> Files;
 798    for (vector<metaIndex *>::const_iterator i = List.begin();
 799         i != List.end();
 800         i++)
 801    {
 802       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 803       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 804            j != Indexes->end();
 805            j++)
 806          Files.push_back (*j);
 807    }
 808
 809    unsigned long EndOfSource = Files.size();
 810    if (_system->AddStatusFiles(Files) == false)
 811       return false;
 812
 813    // Decide if we can write to the files..
 814    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 815    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 816
 817    // Decide if we can write to the cache
 818    bool Writeable = false;
 819    if (CacheFile.empty() == false)
 820       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 821    else
 822       if (SrcCacheFile.empty() == false)
 823          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 824
 825    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 826       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 827
 828    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 829
 830    // Cache is OK, Fin.
 831    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 832    {
 833       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 834       return true;
 835    }
 836
 837    /* At this point we know we need to reconstruct the package cache,
 838       begin. */
 839    SPtr<FileFd> CacheF;
 840    SPtr<DynamicMMap> Map;
 841    if (Writeable == true && CacheFile.empty() == false)
 842    {
 843       unlink(CacheFile.c_str());
 844       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 845       fchmod(CacheF->Fd(),0644);
 846       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 847       if (_error->PendingError() == true)
 848          return false;
 849    }
 850    else
 851    {
 852       // Just build it in memory..
 853       Map = new DynamicMMap(MMap::Public,MapSize);
 854    }
 855
 856    // Lets try the source cache.
 857    unsigned long CurrentSize = 0;
 858    unsigned long TotalSize = 0;
 859    if (CheckValidity(SrcCacheFile,Files.begin(),
 860                      Files.begin()+EndOfSource) == true)
 861    {
 862       // Preload the map with the source cache
 863       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 864       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 865                        SCacheF.Size()) == false)
 866          return false;
 867
 868       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 869
 870       // Build the status cache
 871       pkgCacheGenerator Gen(Map.Get(),&Progress);
 872       if (_error->PendingError() == true)
 873          return false;
 874       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 875                      Files.begin()+EndOfSource,Files.end()) == false)
 876          return false;
 877    }
 878    else
 879    {
 880       TotalSize = ComputeSize(Files.begin(),Files.end());
 881
 882       // Build the source cache
 883       pkgCacheGenerator Gen(Map.Get(),&Progress);
 884       if (_error->PendingError() == true)
 885          return false;
 886       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 887                      Files.begin(),Files.begin()+EndOfSource) == false)
 888          return false;
 889
 890       // Write it back
 891       if (Writeable == true && SrcCacheFile.empty() == false)
 892       {
 893          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 894          if (_error->PendingError() == true)
 895             return false;
 896
 897          fchmod(SCacheF.Fd(),0644);
 898
 899          // Write out the main data
 900          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 901             return _error->Error(_("IO Error saving source cache"));
 902          SCacheF.Sync();
 903
 904          // Write out the proper header
 905          Gen.GetCache().HeaderP->Dirty = false;
 906          if (SCacheF.Seek(0) == false ||
 907              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 908             return _error->Error(_("IO Error saving source cache"));
 909          Gen.GetCache().HeaderP->Dirty = true;
 910          SCacheF.Sync();
 911       }
 912
 913       // Build the status cache
 914       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 915                      Files.begin()+EndOfSource,Files.end()) == false)
 916          return false;
 917    }
 918
 919    if (_error->PendingError() == true)
 920       return false;
 921    if (OutMap != 0)
 922    {
 923       if (CacheF != 0)
 924       {
 925          delete Map.UnGuard();
 926          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 927       }
 928       else
 929       {
 930          *OutMap = Map.UnGuard();
 931       }
 932    }
 933
 934    return true;
 935 }
 936                                                                         /*}}}*/
 937 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 938 // ---------------------------------------------------------------------
 939 /* */
 940 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 941 {
 942    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
 943    vector<pkgIndexFile *> Files;
 944    unsigned long EndOfSource = Files.size();
 945    if (_system->AddStatusFiles(Files) == false)
 946       return false;
 947
 948    SPtr<DynamicMMap> Map;
 949    Map = new DynamicMMap(MMap::Public,MapSize);
 950    unsigned long CurrentSize = 0;
 951    unsigned long TotalSize = 0;
 952
 953    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 954
 955    // Build the status cache
 956    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 957    pkgCacheGenerator Gen(Map.Get(),&Progress);
 958    if (_error->PendingError() == true)
 959       return false;
 960    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 961                   Files.begin()+EndOfSource,Files.end()) == false)
 962       return false;
 963
 964    if (_error->PendingError() == true)
 965       return false;
 966    *OutMap = Map.UnGuard();
 967
 968    return true;
 969 }
 970                                                                         /*}}}*/