apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24
  25 #include <apt-pkg/tagfile.h>
  26
  27 #include <apti18n.h>
  28
  29 #include <vector>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35 #include <system.h>
  36                                                                         /*}}}*/
  37 typedef vector<pkgIndexFile *>::iterator FileIterator;
  38
  39 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* We set the diry flag and make sure that is written to the disk */
  42 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  43                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  44                     FoundFileDeps(0)
  45 {
  46    CurrentFile = 0;
  47    memset(UniqHash,0,sizeof(UniqHash));
  48
  49    if (_error->PendingError() == true)
  50       return;
  51
  52    if (Map.Size() == 0)
  53    {
  54       // Setup the map interface..
  55       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  56       Map.RawAllocate(sizeof(pkgCache::Header));
  57       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  58
  59       // Starting header
  60       *Cache.HeaderP = pkgCache::Header();
  61       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  62       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  63       Cache.ReMap();
  64    }
  65    else
  66    {
  67       // Map directly from the existing file
  68       Cache.ReMap();
  69       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  70       if (Cache.VS != _system->VS)
  71       {
  72          _error->Error(_("Cache has an incompatible versioning system"));
  73          return;
  74       }
  75    }
  76
  77    Cache.HeaderP->Dirty = true;
  78    Map.Sync(0,sizeof(pkgCache::Header));
  79 }
  80                                                                         /*}}}*/
  81 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  82 // ---------------------------------------------------------------------
  83 /* We sync the data then unset the dirty flag in two steps so as to
  84    advoid a problem during a crash */
  85 pkgCacheGenerator::~pkgCacheGenerator()
  86 {
  87    if (_error->PendingError() == true)
  88       return;
  89    if (Map.Sync() == false)
  90       return;
  91
  92    Cache.HeaderP->Dirty = false;
  93    Map.Sync(0,sizeof(pkgCache::Header));
  94 }
  95                                                                         /*}}}*/
  96 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* This provides the generation of the entries in the cache. Each loop
  99    goes through a single package record from the underlying parse engine. */
 100 bool pkgCacheGenerator::MergeList(ListParser &List,
 101                                   pkgCache::VerIterator *OutVer)
 102 {
 103    List.Owner = this;
 104
 105    unsigned int Counter = 0;
 106    while (List.Step() == true)
 107    {
 108       // Get a pointer to the package structure
 109       string PackageName = List.Package();
 110       if (PackageName.empty() == true)
 111          return false;
 112
 113       pkgCache::PkgIterator Pkg;
 114       if (NewPackage(Pkg,PackageName) == false)
 115          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 116       Counter++;
 117       if (Counter % 100 == 0 && Progress != 0)
 118          Progress->Progress(List.Offset());
 119
 120       /* Get a pointer to the version structure. We know the list is sorted
 121          so we use that fact in the search. Insertion of new versions is
 122          done with correct sorting */
 123       string Version = List.Version();
 124       if (Version.empty() == true)
 125       {
 126          // we first process the package, then the descriptions
 127          // (this has the bonus that we get MMap error when we run out
 128          //  of MMap space)
 129          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 130             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 131                                  PackageName.c_str());
 132
 133          // Find the right version to write the description
 134          MD5SumValue CurMd5 = List.Description_md5();
 135          pkgCache::VerIterator Ver = Pkg.VersionList();
 136          map_ptrloc *LastVer = &Pkg->VersionList;
 137
 138          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 139          {
 140             pkgCache::DescIterator Desc = Ver.DescriptionList();
 141             map_ptrloc *LastDesc = &Ver->DescriptionList;
 142             bool duplicate=false;
 143
 144             // don't add a new description if we have one for the given
 145             // md5 && language
 146             for ( ; Desc.end() == false; Desc++)
 147                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 148                    Desc.LanguageCode() == List.DescriptionLanguage())
 149                   duplicate=true;
 150             if(duplicate)
 151                continue;
 152
 153             for (Desc = Ver.DescriptionList();
 154                  Desc.end() == false;
 155                  LastDesc = &Desc->NextDesc, Desc++)
 156             {
 157                if (MD5SumValue(Desc.md5()) == CurMd5)
 158                {
 159                   // Add new description
 160                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 161                   Desc->ParentPkg = Pkg.Index();
 162
 163                   if (NewFileDesc(Desc,List) == false)
 164                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 165                   break;
 166                }
 167             }
 168          }
 169
 170          continue;
 171       }
 172
 173       pkgCache::VerIterator Ver = Pkg.VersionList();
 174       map_ptrloc *LastVer = &Pkg->VersionList;
 175       int Res = 1;
 176       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 177       {
 178          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 179          if (Res >= 0)
 180             break;
 181       }
 182
 183       /* We already have a version for this item, record that we
 184          saw it */
 185       unsigned long Hash = List.VersionHash();
 186       if (Res == 0 && Ver->Hash == Hash)
 187       {
 188          if (List.UsePackage(Pkg,Ver) == false)
 189             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 190                                  PackageName.c_str());
 191
 192          if (NewFileVer(Ver,List) == false)
 193             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 194                                  PackageName.c_str());
 195
 196          // Read only a single record and return
 197          if (OutVer != 0)
 198          {
 199             *OutVer = Ver;
 200             FoundFileDeps |= List.HasFileDeps();
 201             return true;
 202          }
 203
 204          continue;
 205       }
 206
 207       // Skip to the end of the same version set.
 208       if (Res == 0)
 209       {
 210          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 211          {
 212             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 213             if (Res != 0)
 214                break;
 215          }
 216       }
 217
 218       // Add a new version
 219       *LastVer = NewVersion(Ver,Version,*LastVer);
 220       Ver->ParentPkg = Pkg.Index();
 221       Ver->Hash = Hash;
 222
 223       if (List.NewVersion(Ver) == false)
 224          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 225                               PackageName.c_str());
 226
 227       if (List.UsePackage(Pkg,Ver) == false)
 228          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 229                               PackageName.c_str());
 230
 231       if (NewFileVer(Ver,List) == false)
 232          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 233                               PackageName.c_str());
 234
 235       // Read only a single record and return
 236       if (OutVer != 0)
 237       {
 238          *OutVer = Ver;
 239          FoundFileDeps |= List.HasFileDeps();
 240          return true;
 241       }
 242
 243       /* Record the Description data. Description data always exist in
 244          Packages and Translation-* files. */
 245       pkgCache::DescIterator Desc = Ver.DescriptionList();
 246       map_ptrloc *LastDesc = &Ver->DescriptionList;
 247
 248       // Skip to the end of description set
 249       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 250
 251       // Add new description
 252       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 253       Desc->ParentPkg = Pkg.Index();
 254
 255       if (NewFileDesc(Desc,List) == false)
 256          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 257    }
 258
 259    FoundFileDeps |= List.HasFileDeps();
 260
 261    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 262       return _error->Error(_("Wow, you exceeded the number of package "
 263                              "names this APT is capable of."));
 264    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 265       return _error->Error(_("Wow, you exceeded the number of versions "
 266                              "this APT is capable of."));
 267    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 268       return _error->Error(_("Wow, you exceeded the number of descriptions "
 269                              "this APT is capable of."));
 270    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 271       return _error->Error(_("Wow, you exceeded the number of dependencies "
 272                              "this APT is capable of."));
 273    return true;
 274 }
 275                                                                         /*}}}*/
 276 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 277 // ---------------------------------------------------------------------
 278 /* If we found any file depends while parsing the main list we need to
 279    resolve them. Since it is undesired to load the entire list of files
 280    into the cache as virtual packages we do a two stage effort. MergeList
 281    identifies the file depends and this creates Provdies for them by
 282    re-parsing all the indexs. */
 283 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 284 {
 285    List.Owner = this;
 286
 287    unsigned int Counter = 0;
 288    while (List.Step() == true)
 289    {
 290       string PackageName = List.Package();
 291       if (PackageName.empty() == true)
 292          return false;
 293       string Version = List.Version();
 294       if (Version.empty() == true)
 295          continue;
 296
 297       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 298       if (Pkg.end() == true)
 299          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 300                                 PackageName.c_str());
 301       Counter++;
 302       if (Counter % 100 == 0 && Progress != 0)
 303          Progress->Progress(List.Offset());
 304
 305       unsigned long Hash = List.VersionHash();
 306       pkgCache::VerIterator Ver = Pkg.VersionList();
 307       for (; Ver.end() == false; Ver++)
 308       {
 309          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 310          {
 311             if (List.CollectFileProvides(Cache,Ver) == false)
 312                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 313             break;
 314          }
 315       }
 316
 317       if (Ver.end() == true)
 318          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 319    }
 320
 321    return true;
 322 }
 323                                                                         /*}}}*/
 324 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 325 // ---------------------------------------------------------------------
 326 /* This creates a new package structure and adds it to the hash table */
 327 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 328 {
 329    Pkg = Cache.FindPkg(Name);
 330    if (Pkg.end() == false)
 331       return true;
 332
 333    // Get a structure
 334    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 335    if (Package == 0)
 336       return false;
 337
 338    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 339
 340    // Insert it into the hash table
 341    unsigned long Hash = Cache.Hash(Name);
 342    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 343    Cache.HeaderP->HashTable[Hash] = Package;
 344
 345    // Set the name and the ID
 346    Pkg->Name = Map.WriteString(Name);
 347    if (Pkg->Name == 0)
 348       return false;
 349    Pkg->ID = Cache.HeaderP->PackageCount++;
 350
 351    return true;
 352 }
 353                                                                         /*}}}*/
 354 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* */
 357 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 358                                    ListParser &List)
 359 {
 360    if (CurrentFile == 0)
 361       return true;
 362
 363    // Get a structure
 364    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 365    if (VerFile == 0)
 366       return 0;
 367
 368    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 369    VF->File = CurrentFile - Cache.PkgFileP;
 370
 371    // Link it to the end of the list
 372    map_ptrloc *Last = &Ver->FileList;
 373    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 374       Last = &V->NextFile;
 375    VF->NextFile = *Last;
 376    *Last = VF.Index();
 377
 378    VF->Offset = List.Offset();
 379    VF->Size = List.Size();
 380    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 381       Cache.HeaderP->MaxVerFileSize = VF->Size;
 382    Cache.HeaderP->VerFileCount++;
 383
 384    return true;
 385 }
 386                                                                         /*}}}*/
 387 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 388 // ---------------------------------------------------------------------
 389 /* This puts a version structure in the linked list */
 390 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 391                                             const string &VerStr,
 392                                             unsigned long Next)
 393 {
 394    // Get a structure
 395    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 396    if (Version == 0)
 397       return 0;
 398
 399    // Fill it in
 400    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 401    Ver->NextVer = Next;
 402    Ver->ID = Cache.HeaderP->VersionCount++;
 403    Ver->VerStr = Map.WriteString(VerStr);
 404    if (Ver->VerStr == 0)
 405       return 0;
 406
 407    return Version;
 408 }
 409                                                                         /*}}}*/
 410 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 411 // ---------------------------------------------------------------------
 412 /* */
 413 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 414                                    ListParser &List)
 415 {
 416    if (CurrentFile == 0)
 417       return true;
 418
 419    // Get a structure
 420    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 421    if (DescFile == 0)
 422       return 0;
 423
 424    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 425    DF->File = CurrentFile - Cache.PkgFileP;
 426
 427    // Link it to the end of the list
 428    map_ptrloc *Last = &Desc->FileList;
 429    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 430       Last = &D->NextFile;
 431
 432    DF->NextFile = *Last;
 433    *Last = DF.Index();
 434
 435    DF->Offset = List.Offset();
 436    DF->Size = List.Size();
 437    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 438       Cache.HeaderP->MaxDescFileSize = DF->Size;
 439    Cache.HeaderP->DescFileCount++;
 440
 441    return true;
 442 }
 443                                                                         /*}}}*/
 444 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 445 // ---------------------------------------------------------------------
 446 /* This puts a description structure in the linked list */
 447 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 448                                             const string &Lang,
 449                                             const MD5SumValue &md5sum,
 450                                             map_ptrloc Next)
 451 {
 452    // Get a structure
 453    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 454    if (Description == 0)
 455       return 0;
 456
 457    // Fill it in
 458    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 459    Desc->NextDesc = Next;
 460    Desc->ID = Cache.HeaderP->DescriptionCount++;
 461    Desc->language_code = Map.WriteString(Lang);
 462    Desc->md5sum = Map.WriteString(md5sum.Value());
 463
 464    return Description;
 465 }
 466                                                                         /*}}}*/
 467 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 468 // ---------------------------------------------------------------------
 469 /* This creates a dependency element in the tree. It is linked to the
 470    version and to the package that it is pointing to. */
 471 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 472                                                const string &PackageName,
 473                                                const string &Version,
 474                                                unsigned int Op,
 475                                                unsigned int Type)
 476 {
 477    pkgCache &Cache = Owner->Cache;
 478
 479    // Get a structure
 480    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 481    if (Dependency == 0)
 482       return false;
 483
 484    // Fill it in
 485    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 486    Dep->ParentVer = Ver.Index();
 487    Dep->Type = Type;
 488    Dep->CompareOp = Op;
 489    Dep->ID = Cache.HeaderP->DependsCount++;
 490
 491    // Locate the target package
 492    pkgCache::PkgIterator Pkg;
 493    if (Owner->NewPackage(Pkg,PackageName) == false)
 494       return false;
 495
 496    // Probe the reverse dependency list for a version string that matches
 497    if (Version.empty() == false)
 498    {
 499 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 500          if (I->Version != 0 && I.TargetVer() == Version)
 501             Dep->Version = I->Version;*/
 502       if (Dep->Version == 0)
 503          if ((Dep->Version = WriteString(Version)) == 0)
 504             return false;
 505    }
 506
 507    // Link it to the package
 508    Dep->Package = Pkg.Index();
 509    Dep->NextRevDepends = Pkg->RevDepends;
 510    Pkg->RevDepends = Dep.Index();
 511
 512    /* Link it to the version (at the end of the list)
 513       Caching the old end point speeds up generation substantially */
 514    if (OldDepVer != Ver)
 515    {
 516       OldDepLast = &Ver->DependsList;
 517       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 518          OldDepLast = &D->NextDepends;
 519       OldDepVer = Ver;
 520    }
 521
 522    // Is it a file dependency?
 523    if (PackageName[0] == '/')
 524       FoundFileDeps = true;
 525
 526    Dep->NextDepends = *OldDepLast;
 527    *OldDepLast = Dep.Index();
 528    OldDepLast = &Dep->NextDepends;
 529
 530    return true;
 531 }
 532                                                                         /*}}}*/
 533 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 534 // ---------------------------------------------------------------------
 535 /* */
 536 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 537                                                 const string &PackageName,
 538                                                 const string &Version)
 539 {
 540    pkgCache &Cache = Owner->Cache;
 541
 542    // We do not add self referencing provides
 543    if (Ver.ParentPkg().Name() == PackageName)
 544       return true;
 545
 546    // Get a structure
 547    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 548    if (Provides == 0)
 549       return false;
 550    Cache.HeaderP->ProvidesCount++;
 551
 552    // Fill it in
 553    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 554    Prv->Version = Ver.Index();
 555    Prv->NextPkgProv = Ver->ProvidesList;
 556    Ver->ProvidesList = Prv.Index();
 557    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 558       return false;
 559
 560    // Locate the target package
 561    pkgCache::PkgIterator Pkg;
 562    if (Owner->NewPackage(Pkg,PackageName) == false)
 563       return false;
 564
 565    // Link it to the package
 566    Prv->ParentPkg = Pkg.Index();
 567    Prv->NextProvides = Pkg->ProvidesList;
 568    Pkg->ProvidesList = Prv.Index();
 569
 570    return true;
 571 }
 572                                                                         /*}}}*/
 573 // ListParser::NewTag - Create a Tag element                            /*{{{*/
 574 // ---------------------------------------------------------------------
 575 /* */
 576 bool pkgCacheGenerator::ListParser::NewTag(pkgCache::PkgIterator Pkg,
 577                                            const char *NameStart,
 578                                            unsigned int NameSize)
 579 {
 580    pkgCache &Cache = Owner->Cache;
 581
 582    // Get a structure
 583    unsigned long Tagg = Owner->Map.Allocate(sizeof(pkgCache::Tag));
 584    if (Tagg == 0)
 585       return false;
 586    Cache.HeaderP->TagCount++;
 587
 588    // Fill it in
 589    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
 590    Tg->Name = WriteString(NameStart,NameSize);
 591    if (Tg->Name == 0)
 592       return false;
 593    Tg->NextTag = Pkg->TagList;
 594    Pkg->TagList = Tg.Index();
 595
 596    return true;
 597 }
 598                                                                         /*}}}*/
 599 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* This is used to select which file is to be associated with all newly
 602    added versions. The caller is responsible for setting the IMS fields. */
 603 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 604                                    const pkgIndexFile &Index,
 605                                    unsigned long Flags)
 606 {
 607    // Get some space for the structure
 608    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 609    if (CurrentFile == Cache.PkgFileP)
 610       return false;
 611
 612    // Fill it in
 613    CurrentFile->FileName = Map.WriteString(File);
 614    CurrentFile->Site = WriteUniqString(Site);
 615    CurrentFile->NextFile = Cache.HeaderP->FileList;
 616    CurrentFile->Flags = Flags;
 617    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 618    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 619    PkgFileName = File;
 620    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 621    Cache.HeaderP->PackageFileCount++;
 622
 623    if (CurrentFile->FileName == 0)
 624       return false;
 625
 626    if (Progress != 0)
 627       Progress->SubProgress(Index.Size());
 628    return true;
 629 }
 630                                                                         /*}}}*/
 631 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 632 // ---------------------------------------------------------------------
 633 /* This is used to create handles to strings. Given the same text it
 634    always returns the same number */
 635 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 636                                                  unsigned int Size)
 637 {
 638    /* We use a very small transient hash table here, this speeds up generation
 639       by a fair amount on slower machines */
 640    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 641    if (Bucket != 0 &&
 642        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 643       return Bucket->String;
 644
 645    // Search for an insertion point
 646    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 647    int Res = 1;
 648    map_ptrloc *Last = &Cache.HeaderP->StringList;
 649    for (; I != Cache.StringItemP; Last = &I->NextItem,
 650         I = Cache.StringItemP + I->NextItem)
 651    {
 652       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 653       if (Res >= 0)
 654          break;
 655    }
 656
 657    // Match
 658    if (Res == 0)
 659    {
 660       Bucket = I;
 661       return I->String;
 662    }
 663
 664    // Get a structure
 665    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 666    if (Item == 0)
 667       return 0;
 668
 669    // Fill in the structure
 670    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 671    ItemP->NextItem = I - Cache.StringItemP;
 672    *Last = Item;
 673    ItemP->String = Map.WriteString(S,Size);
 674    if (ItemP->String == 0)
 675       return 0;
 676
 677    Bucket = ItemP;
 678    return ItemP->String;
 679 }
 680                                                                         /*}}}*/
 681
 682 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 683 // ---------------------------------------------------------------------
 684 /* This just verifies that each file in the list of index files exists,
 685    has matching attributes with the cache and the cache does not have
 686    any extra files. */
 687 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 688                           FileIterator End,MMap **OutMap = 0)
 689 {
 690    // No file, certainly invalid
 691    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 692       return false;
 693
 694    // Map it
 695    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 696    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 697    pkgCache Cache(Map);
 698    if (_error->PendingError() == true || Map->Size() == 0)
 699    {
 700       _error->Discard();
 701       return false;
 702    }
 703
 704    /* Now we check every index file, see if it is in the cache,
 705       verify the IMS data and check that it is on the disk too.. */
 706    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 707    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 708    for (; Start != End; Start++)
 709    {
 710       if ((*Start)->HasPackages() == false)
 711          continue;
 712
 713       if ((*Start)->Exists() == false)
 714       {
 715 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 716          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 717                           (*Start)->Describe().c_str());
 718 #endif
 719          continue;
 720       }
 721
 722       // FindInCache is also expected to do an IMS check.
 723       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 724       if (File.end() == true)
 725          return false;
 726
 727       Visited[File->ID] = true;
 728    }
 729
 730    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 731       if (Visited[I] == false)
 732          return false;
 733
 734    if (_error->PendingError() == true)
 735    {
 736       _error->Discard();
 737       return false;
 738    }
 739
 740    if (OutMap != 0)
 741       *OutMap = Map.UnGuard();
 742    return true;
 743 }
 744                                                                         /*}}}*/
 745 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 746 // ---------------------------------------------------------------------
 747 /* Size is kind of an abstract notion that is only used for the progress
 748    meter */
 749 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 750 {
 751    unsigned long TotalSize = 0;
 752    for (; Start != End; Start++)
 753    {
 754       if ((*Start)->HasPackages() == false)
 755          continue;
 756       TotalSize += (*Start)->Size();
 757    }
 758    return TotalSize;
 759 }
 760                                                                         /*}}}*/
 761 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 762 // ---------------------------------------------------------------------
 763 /* */
 764 static bool BuildCache(pkgCacheGenerator &Gen,
 765                        OpProgress &Progress,
 766                        unsigned long &CurrentSize,unsigned long TotalSize,
 767                        FileIterator Start, FileIterator End)
 768 {
 769    FileIterator I;
 770    for (I = Start; I != End; I++)
 771    {
 772       if ((*I)->HasPackages() == false)
 773          continue;
 774
 775       if ((*I)->Exists() == false)
 776          continue;
 777
 778       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 779       {
 780          _error->Warning("Duplicate sources.list entry %s",
 781                          (*I)->Describe().c_str());
 782          continue;
 783       }
 784
 785       unsigned long Size = (*I)->Size();
 786       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 787       CurrentSize += Size;
 788
 789       if ((*I)->Merge(Gen,Progress) == false)
 790          return false;
 791    }
 792
 793    if (Gen.HasFileDeps() == true)
 794    {
 795       Progress.Done();
 796       TotalSize = ComputeSize(Start, End);
 797       CurrentSize = 0;
 798       for (I = Start; I != End; I++)
 799       {
 800          unsigned long Size = (*I)->Size();
 801          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 802          CurrentSize += Size;
 803          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 804             return false;
 805       }
 806    }
 807
 808    return true;
 809 }
 810                                                                         /*}}}*/
 811 // MakeStatusCache - Construct the status cache                         /*{{{*/
 812 // ---------------------------------------------------------------------
 813 /* This makes sure that the status cache (the cache that has all
 814    index files from the sources list and all local ones) is ready
 815    to be mmaped. If OutMap is not zero then a MMap object representing
 816    the cache will be stored there. This is pretty much mandetory if you
 817    are using AllowMem. AllowMem lets the function be run as non-root
 818    where it builds the cache 'fast' into a memory buffer. */
 819 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 820                         MMap **OutMap,bool AllowMem)
 821 {
 822    unsigned long MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 823
 824    vector<pkgIndexFile *> Files;
 825    for (vector<metaIndex *>::const_iterator i = List.begin();
 826         i != List.end();
 827         i++)
 828    {
 829       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 830       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 831            j != Indexes->end();
 832            j++)
 833          Files.push_back (*j);
 834    }
 835
 836    unsigned long EndOfSource = Files.size();
 837    if (_system->AddStatusFiles(Files) == false)
 838       return false;
 839
 840    // Decide if we can write to the files..
 841    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 842    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 843
 844    // Decide if we can write to the cache
 845    bool Writeable = false;
 846    if (CacheFile.empty() == false)
 847       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 848    else
 849       if (SrcCacheFile.empty() == false)
 850          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 851
 852    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 853       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 854
 855    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 856
 857    // Cache is OK, Fin.
 858    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 859    {
 860       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 861       return true;
 862    }
 863
 864    /* At this point we know we need to reconstruct the package cache,
 865       begin. */
 866    SPtr<FileFd> CacheF;
 867    SPtr<DynamicMMap> Map;
 868    if (Writeable == true && CacheFile.empty() == false)
 869    {
 870       unlink(CacheFile.c_str());
 871       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 872       fchmod(CacheF->Fd(),0644);
 873       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 874       if (_error->PendingError() == true)
 875          return false;
 876    }
 877    else
 878    {
 879       // Just build it in memory..
 880       Map = new DynamicMMap(MMap::Public,MapSize);
 881    }
 882
 883    // Lets try the source cache.
 884    unsigned long CurrentSize = 0;
 885    unsigned long TotalSize = 0;
 886    if (CheckValidity(SrcCacheFile,Files.begin(),
 887                      Files.begin()+EndOfSource) == true)
 888    {
 889       // Preload the map with the source cache
 890       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 891       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 892                        SCacheF.Size()) == false)
 893          return false;
 894
 895       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 896
 897       // Build the status cache
 898       pkgCacheGenerator Gen(Map.Get(),&Progress);
 899       if (_error->PendingError() == true)
 900          return false;
 901       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 902                      Files.begin()+EndOfSource,Files.end()) == false)
 903          return false;
 904    }
 905    else
 906    {
 907       TotalSize = ComputeSize(Files.begin(),Files.end());
 908
 909       // Build the source cache
 910       pkgCacheGenerator Gen(Map.Get(),&Progress);
 911       if (_error->PendingError() == true)
 912          return false;
 913       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 914                      Files.begin(),Files.begin()+EndOfSource) == false)
 915          return false;
 916
 917       // Write it back
 918       if (Writeable == true && SrcCacheFile.empty() == false)
 919       {
 920          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 921          if (_error->PendingError() == true)
 922             return false;
 923
 924          fchmod(SCacheF.Fd(),0644);
 925
 926          // Write out the main data
 927          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 928             return _error->Error(_("IO Error saving source cache"));
 929          SCacheF.Sync();
 930
 931          // Write out the proper header
 932          Gen.GetCache().HeaderP->Dirty = false;
 933          if (SCacheF.Seek(0) == false ||
 934              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 935             return _error->Error(_("IO Error saving source cache"));
 936          Gen.GetCache().HeaderP->Dirty = true;
 937          SCacheF.Sync();
 938       }
 939
 940       // Build the status cache
 941       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 942                      Files.begin()+EndOfSource,Files.end()) == false)
 943          return false;
 944    }
 945
 946    if (_error->PendingError() == true)
 947       return false;
 948    if (OutMap != 0)
 949    {
 950       if (CacheF != 0)
 951       {
 952          delete Map.UnGuard();
 953          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 954       }
 955       else
 956       {
 957          *OutMap = Map.UnGuard();
 958       }
 959    }
 960
 961    return true;
 962 }
 963                                                                         /*}}}*/
 964 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 965 // ---------------------------------------------------------------------
 966 /* */
 967 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 968 {
 969    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
 970    vector<pkgIndexFile *> Files;
 971    unsigned long EndOfSource = Files.size();
 972    if (_system->AddStatusFiles(Files) == false)
 973       return false;
 974
 975    SPtr<DynamicMMap> Map;
 976    Map = new DynamicMMap(MMap::Public,MapSize);
 977    unsigned long CurrentSize = 0;
 978    unsigned long TotalSize = 0;
 979
 980    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 981
 982    // Build the status cache
 983    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 984    pkgCacheGenerator Gen(Map.Get(),&Progress);
 985    if (_error->PendingError() == true)
 986       return false;
 987    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 988                   Files.begin()+EndOfSource,Files.end()) == false)
 989       return false;
 990
 991    if (_error->PendingError() == true)
 992       return false;
 993    *OutMap = Map.UnGuard();
 994
 995    return true;
 996 }
 997                                                                         /*}}}*/