apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24
  25 #include <apt-pkg/tagfile.h>
  26
  27 #include <apti18n.h>
  28
  29 #include <vector>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35 #include <system.h>
  36                                                                         /*}}}*/
  37 typedef vector<pkgIndexFile *>::iterator FileIterator;
  38 uint32_t hashlittle( const void *key, size_t length, uint32_t initval);
  39
  40 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* We set the diry flag and make sure that is written to the disk */
  43 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  44                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  45                     FoundFileDeps(0)
  46 {
  47    CurrentFile = 0;
  48    memset(UniqHash,0,sizeof(UniqHash));
  49
  50    if (_error->PendingError() == true)
  51       return;
  52
  53    if (Map.Size() == 0)
  54    {
  55       // Setup the map interface..
  56       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  57       Map.RawAllocate(sizeof(pkgCache::Header));
  58       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  59
  60       // Starting header
  61       *Cache.HeaderP = pkgCache::Header();
  62       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  63       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  64       Cache.ReMap();
  65    }
  66    else
  67    {
  68       // Map directly from the existing file
  69       Cache.ReMap();
  70       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  71       if (Cache.VS != _system->VS)
  72       {
  73          _error->Error(_("Cache has an incompatible versioning system"));
  74          return;
  75       }
  76    }
  77
  78    Cache.HeaderP->Dirty = true;
  79    Map.Sync(0,sizeof(pkgCache::Header));
  80 }
  81                                                                         /*}}}*/
  82 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  83 // ---------------------------------------------------------------------
  84 /* We sync the data then unset the dirty flag in two steps so as to
  85    advoid a problem during a crash */
  86 pkgCacheGenerator::~pkgCacheGenerator()
  87 {
  88    if (_error->PendingError() == true)
  89       return;
  90    if (Map.Sync() == false)
  91       return;
  92
  93    Cache.HeaderP->Dirty = false;
  94    Map.Sync(0,sizeof(pkgCache::Header));
  95 }
  96                                                                         /*}}}*/
  97 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* This provides the generation of the entries in the cache. Each loop
 100    goes through a single package record from the underlying parse engine. */
 101 bool pkgCacheGenerator::MergeList(ListParser &List,
 102                                   pkgCache::VerIterator *OutVer)
 103 {
 104    List.Owner = this;
 105
 106    unsigned int Counter = 0;
 107    while (List.Step() == true)
 108    {
 109       // Get a pointer to the package structure
 110       string PackageName = List.Package();
 111       if (PackageName.empty() == true)
 112          return false;
 113
 114       pkgCache::PkgIterator Pkg;
 115       if (NewPackage(Pkg,PackageName) == false)
 116          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 117       Counter++;
 118       if (Counter % 100 == 0 && Progress != 0)
 119          Progress->Progress(List.Offset());
 120
 121       /* Get a pointer to the version structure. We know the list is sorted
 122          so we use that fact in the search. Insertion of new versions is
 123          done with correct sorting */
 124       string Version = List.Version();
 125       if (Version.empty() == true)
 126       {
 127          // we first process the package, then the descriptions
 128          // (this has the bonus that we get MMap error when we run out
 129          //  of MMap space)
 130          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 131             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 132                                  PackageName.c_str());
 133
 134          // Find the right version to write the description
 135          MD5SumValue CurMd5 = List.Description_md5();
 136          pkgCache::VerIterator Ver = Pkg.VersionList();
 137          map_ptrloc *LastVer = &Pkg->VersionList;
 138
 139          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 140          {
 141             pkgCache::DescIterator Desc = Ver.DescriptionList();
 142             map_ptrloc *LastDesc = &Ver->DescriptionList;
 143             bool duplicate=false;
 144
 145             // don't add a new description if we have one for the given
 146             // md5 && language
 147             for ( ; Desc.end() == false; Desc++)
 148                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 149                    Desc.LanguageCode() == List.DescriptionLanguage())
 150                   duplicate=true;
 151             if(duplicate)
 152                continue;
 153
 154             for (Desc = Ver.DescriptionList();
 155                  Desc.end() == false;
 156                  LastDesc = &Desc->NextDesc, Desc++)
 157             {
 158                if (MD5SumValue(Desc.md5()) == CurMd5)
 159                {
 160                   // Add new description
 161                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 162                   Desc->ParentPkg = Pkg.Index();
 163
 164                   if (NewFileDesc(Desc,List) == false)
 165                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 166                   break;
 167                }
 168             }
 169          }
 170
 171          continue;
 172       }
 173
 174       pkgCache::VerIterator Ver = Pkg.VersionList();
 175       map_ptrloc *LastVer = &Pkg->VersionList;
 176       int Res = 1;
 177       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 178       {
 179          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 180          if (Res >= 0)
 181             break;
 182       }
 183
 184       /* We already have a version for this item, record that we
 185          saw it */
 186       unsigned long Hash = List.VersionHash();
 187       if (Res == 0 && Ver->Hash == Hash)
 188       {
 189          if (List.UsePackage(Pkg,Ver) == false)
 190             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 191                                  PackageName.c_str());
 192
 193          if (NewFileVer(Ver,List) == false)
 194             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 195                                  PackageName.c_str());
 196
 197          // Read only a single record and return
 198          if (OutVer != 0)
 199          {
 200             *OutVer = Ver;
 201             FoundFileDeps |= List.HasFileDeps();
 202             return true;
 203          }
 204
 205          continue;
 206       }
 207
 208       // Skip to the end of the same version set.
 209       if (Res == 0)
 210       {
 211          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 212          {
 213             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 214             if (Res != 0)
 215                break;
 216          }
 217       }
 218
 219       // Add a new version
 220       *LastVer = NewVersion(Ver,Version,*LastVer);
 221       Ver->ParentPkg = Pkg.Index();
 222       Ver->Hash = Hash;
 223
 224       if (List.NewVersion(Ver) == false)
 225          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 226                               PackageName.c_str());
 227
 228       if (List.UsePackage(Pkg,Ver) == false)
 229          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 230                               PackageName.c_str());
 231
 232       if (NewFileVer(Ver,List) == false)
 233          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 234                               PackageName.c_str());
 235
 236       // Read only a single record and return
 237       if (OutVer != 0)
 238       {
 239          *OutVer = Ver;
 240          FoundFileDeps |= List.HasFileDeps();
 241          return true;
 242       }
 243
 244       /* Record the Description data. Description data always exist in
 245          Packages and Translation-* files. */
 246       pkgCache::DescIterator Desc = Ver.DescriptionList();
 247       map_ptrloc *LastDesc = &Ver->DescriptionList;
 248
 249       // Skip to the end of description set
 250       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 251
 252       // Add new description
 253       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 254       Desc->ParentPkg = Pkg.Index();
 255
 256       if (NewFileDesc(Desc,List) == false)
 257          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 258    }
 259
 260    FoundFileDeps |= List.HasFileDeps();
 261
 262    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 263       return _error->Error(_("Wow, you exceeded the number of package "
 264                              "names this APT is capable of."));
 265    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 266       return _error->Error(_("Wow, you exceeded the number of versions "
 267                              "this APT is capable of."));
 268    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 269       return _error->Error(_("Wow, you exceeded the number of descriptions "
 270                              "this APT is capable of."));
 271    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 272       return _error->Error(_("Wow, you exceeded the number of dependencies "
 273                              "this APT is capable of."));
 274    return true;
 275 }
 276                                                                         /*}}}*/
 277 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 278 // ---------------------------------------------------------------------
 279 /* If we found any file depends while parsing the main list we need to
 280    resolve them. Since it is undesired to load the entire list of files
 281    into the cache as virtual packages we do a two stage effort. MergeList
 282    identifies the file depends and this creates Provdies for them by
 283    re-parsing all the indexs. */
 284 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 285 {
 286    List.Owner = this;
 287
 288    unsigned int Counter = 0;
 289    while (List.Step() == true)
 290    {
 291       string PackageName = List.Package();
 292       if (PackageName.empty() == true)
 293          return false;
 294       string Version = List.Version();
 295       if (Version.empty() == true)
 296          continue;
 297
 298       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 299       if (Pkg.end() == true)
 300          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 301                                 PackageName.c_str());
 302       Counter++;
 303       if (Counter % 100 == 0 && Progress != 0)
 304          Progress->Progress(List.Offset());
 305
 306       unsigned long Hash = List.VersionHash();
 307       pkgCache::VerIterator Ver = Pkg.VersionList();
 308       for (; Ver.end() == false; Ver++)
 309       {
 310          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 311          {
 312             if (List.CollectFileProvides(Cache,Ver) == false)
 313                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 314             break;
 315          }
 316       }
 317
 318       if (Ver.end() == true)
 319          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 320    }
 321
 322    return true;
 323 }
 324                                                                         /*}}}*/
 325 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 326 // ---------------------------------------------------------------------
 327 /* This creates a new package structure and adds it to the hash table */
 328 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 329 {
 330    Pkg = Cache.FindPkg(Name);
 331    if (Pkg.end() == false)
 332       return true;
 333
 334    // Get a structure
 335    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 336    if (Package == 0)
 337       return false;
 338
 339    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 340
 341    // Insert it into the hash table
 342    unsigned long Hash = Cache.Hash(Name);
 343    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 344    Cache.HeaderP->HashTable[Hash] = Package;
 345
 346    // Set the name and the ID
 347    Pkg->Name = Map.WriteString(Name);
 348    if (Pkg->Name == 0)
 349       return false;
 350    Pkg->ID = Cache.HeaderP->PackageCount++;
 351
 352    return true;
 353 }
 354                                                                         /*}}}*/
 355 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 356 // ---------------------------------------------------------------------
 357 /* */
 358 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 359                                    ListParser &List)
 360 {
 361    if (CurrentFile == 0)
 362       return true;
 363
 364    // Get a structure
 365    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 366    if (VerFile == 0)
 367       return 0;
 368
 369    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 370    VF->File = CurrentFile - Cache.PkgFileP;
 371
 372    // Link it to the end of the list
 373    map_ptrloc *Last = &Ver->FileList;
 374    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 375       Last = &V->NextFile;
 376    VF->NextFile = *Last;
 377    *Last = VF.Index();
 378
 379    VF->Offset = List.Offset();
 380    VF->Size = List.Size();
 381    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 382       Cache.HeaderP->MaxVerFileSize = VF->Size;
 383    Cache.HeaderP->VerFileCount++;
 384
 385    return true;
 386 }
 387                                                                         /*}}}*/
 388 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 389 // ---------------------------------------------------------------------
 390 /* This puts a version structure in the linked list */
 391 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 392                                             const string &VerStr,
 393                                             unsigned long Next)
 394 {
 395    // Get a structure
 396    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 397    if (Version == 0)
 398       return 0;
 399
 400    // Fill it in
 401    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 402    Ver->NextVer = Next;
 403    Ver->ID = Cache.HeaderP->VersionCount++;
 404    Ver->VerStr = Map.WriteString(VerStr);
 405    if (Ver->VerStr == 0)
 406       return 0;
 407
 408    return Version;
 409 }
 410                                                                         /*}}}*/
 411 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 412 // ---------------------------------------------------------------------
 413 /* */
 414 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 415                                    ListParser &List)
 416 {
 417    if (CurrentFile == 0)
 418       return true;
 419
 420    // Get a structure
 421    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 422    if (DescFile == 0)
 423       return 0;
 424
 425    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 426    DF->File = CurrentFile - Cache.PkgFileP;
 427
 428    // Link it to the end of the list
 429    map_ptrloc *Last = &Desc->FileList;
 430    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 431       Last = &D->NextFile;
 432
 433    DF->NextFile = *Last;
 434    *Last = DF.Index();
 435
 436    DF->Offset = List.Offset();
 437    DF->Size = List.Size();
 438    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 439       Cache.HeaderP->MaxDescFileSize = DF->Size;
 440    Cache.HeaderP->DescFileCount++;
 441
 442    return true;
 443 }
 444                                                                         /*}}}*/
 445 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 446 // ---------------------------------------------------------------------
 447 /* This puts a description structure in the linked list */
 448 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 449                                             const string &Lang,
 450                                             const MD5SumValue &md5sum,
 451                                             map_ptrloc Next)
 452 {
 453    // Get a structure
 454    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 455    if (Description == 0)
 456       return 0;
 457
 458    // Fill it in
 459    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 460    Desc->NextDesc = Next;
 461    Desc->ID = Cache.HeaderP->DescriptionCount++;
 462    Desc->language_code = Map.WriteString(Lang);
 463    Desc->md5sum = Map.WriteString(md5sum.Value());
 464
 465    return Description;
 466 }
 467                                                                         /*}}}*/
 468 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 469 // ---------------------------------------------------------------------
 470 /* This creates a dependency element in the tree. It is linked to the
 471    version and to the package that it is pointing to. */
 472 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 473                                                const string &PackageName,
 474                                                const string &Version,
 475                                                unsigned int Op,
 476                                                unsigned int Type)
 477 {
 478    pkgCache &Cache = Owner->Cache;
 479
 480    // Get a structure
 481    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 482    if (Dependency == 0)
 483       return false;
 484
 485    // Fill it in
 486    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 487    Dep->ParentVer = Ver.Index();
 488    Dep->Type = Type;
 489    Dep->CompareOp = Op;
 490    Dep->ID = Cache.HeaderP->DependsCount++;
 491
 492    // Locate the target package
 493    pkgCache::PkgIterator Pkg;
 494    if (Owner->NewPackage(Pkg,PackageName) == false)
 495       return false;
 496
 497    // Probe the reverse dependency list for a version string that matches
 498    if (Version.empty() == false)
 499    {
 500 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 501          if (I->Version != 0 && I.TargetVer() == Version)
 502             Dep->Version = I->Version;*/
 503       if (Dep->Version == 0)
 504          if ((Dep->Version = WriteString(Version)) == 0)
 505             return false;
 506    }
 507
 508    // Link it to the package
 509    Dep->Package = Pkg.Index();
 510    Dep->NextRevDepends = Pkg->RevDepends;
 511    Pkg->RevDepends = Dep.Index();
 512
 513    /* Link it to the version (at the end of the list)
 514       Caching the old end point speeds up generation substantially */
 515    if (OldDepVer != Ver)
 516    {
 517       OldDepLast = &Ver->DependsList;
 518       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 519          OldDepLast = &D->NextDepends;
 520       OldDepVer = Ver;
 521    }
 522
 523    // Is it a file dependency?
 524    if (PackageName[0] == '/')
 525       FoundFileDeps = true;
 526
 527    Dep->NextDepends = *OldDepLast;
 528    *OldDepLast = Dep.Index();
 529    OldDepLast = &Dep->NextDepends;
 530
 531    return true;
 532 }
 533                                                                         /*}}}*/
 534 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 535 // ---------------------------------------------------------------------
 536 /* */
 537 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 538                                                 const string &PackageName,
 539                                                 const string &Version)
 540 {
 541    pkgCache &Cache = Owner->Cache;
 542
 543    // We do not add self referencing provides
 544    if (Ver.ParentPkg().Name() == PackageName)
 545       return true;
 546
 547    // Get a structure
 548    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 549    if (Provides == 0)
 550       return false;
 551    Cache.HeaderP->ProvidesCount++;
 552
 553    // Fill it in
 554    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 555    Prv->Version = Ver.Index();
 556    Prv->NextPkgProv = Ver->ProvidesList;
 557    Ver->ProvidesList = Prv.Index();
 558    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 559       return false;
 560
 561    // Locate the target package
 562    pkgCache::PkgIterator Pkg;
 563    if (Owner->NewPackage(Pkg,PackageName) == false)
 564       return false;
 565
 566    // Link it to the package
 567    Prv->ParentPkg = Pkg.Index();
 568    Prv->NextProvides = Pkg->ProvidesList;
 569    Pkg->ProvidesList = Prv.Index();
 570
 571    return true;
 572 }
 573                                                                         /*}}}*/
 574 // ListParser::NewTag - Create a Tag element                            /*{{{*/
 575 // ---------------------------------------------------------------------
 576 /* */
 577 bool pkgCacheGenerator::ListParser::NewTag(pkgCache::PkgIterator Pkg,
 578                                            const char *NameStart,
 579                                            unsigned int NameSize)
 580 {
 581    pkgCache &Cache = Owner->Cache;
 582
 583    // Get a structure
 584    unsigned long Tagg = Owner->Map.Allocate(sizeof(pkgCache::Tag));
 585    if (Tagg == 0)
 586       return false;
 587    Cache.HeaderP->TagCount++;
 588
 589    // Fill it in
 590    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
 591    Tg->Name = WriteString(NameStart,NameSize);
 592    if (Tg->Name == 0)
 593       return false;
 594    Tg->NextTag = Pkg->TagList;
 595    Pkg->TagList = Tg.Index();
 596
 597    return true;
 598 }
 599                                                                         /*}}}*/
 600 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* This is used to select which file is to be associated with all newly
 603    added versions. The caller is responsible for setting the IMS fields. */
 604 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 605                                    const pkgIndexFile &Index,
 606                                    unsigned long Flags)
 607 {
 608    // Get some space for the structure
 609    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 610    if (CurrentFile == Cache.PkgFileP)
 611       return false;
 612
 613    // Fill it in
 614    CurrentFile->FileName = Map.WriteString(File);
 615    CurrentFile->Site = WriteUniqString(Site);
 616    CurrentFile->NextFile = Cache.HeaderP->FileList;
 617    CurrentFile->Flags = Flags;
 618    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 619    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 620    PkgFileName = File;
 621    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 622    Cache.HeaderP->PackageFileCount++;
 623
 624    if (CurrentFile->FileName == 0)
 625       return false;
 626
 627    if (Progress != 0)
 628       Progress->SubProgress(Index.Size());
 629    return true;
 630 }
 631                                                                         /*}}}*/
 632 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 633 // ---------------------------------------------------------------------
 634 /* This is used to create handles to strings. Given the same text it
 635    always returns the same number */
 636 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 637                                                  unsigned int Size)
 638 {
 639    uint32_t hash = hashlittle(S, Size, 0xdeadbeef);
 640
 641    /* We use a VERY LARGE INTRANSIENT hash table here, this speeds up generation
 642       by AN INSANE amount on ALL machines */
 643    pkgCache::StringItem **Bucket2;
 644    while (true) {
 645       Bucket2 = &UniqHash[hash % _count(UniqHash)];
 646       if (*Bucket2 == NULL)
 647          break;
 648       if (stringcmp(S,S+Size,Cache.StrP + (*Bucket2)->String) == 0)
 649          return (*Bucket2)->String;
 650       hash += 7;
 651    }
 652
 653    pkgCache::StringItem *&Bucket = *Bucket2;
 654    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 655    map_ptrloc *Last = &Cache.HeaderP->StringList;
 656
 657    // Get a structure
 658    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 659    if (Item == 0)
 660       return 0;
 661
 662    // Fill in the structure
 663    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 664    ItemP->NextItem = I - Cache.StringItemP;
 665    *Last = Item;
 666    ItemP->String = Map.WriteString(S,Size);
 667    if (ItemP->String == 0)
 668       return 0;
 669
 670    Bucket = ItemP;
 671    return ItemP->String;
 672 }
 673                                                                         /*}}}*/
 674
 675 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 676 // ---------------------------------------------------------------------
 677 /* This just verifies that each file in the list of index files exists,
 678    has matching attributes with the cache and the cache does not have
 679    any extra files. */
 680 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 681                           FileIterator End,MMap **OutMap = 0)
 682 {
 683    // No file, certainly invalid
 684    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 685       return false;
 686
 687    // Map it
 688    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 689    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 690    pkgCache Cache(Map);
 691    if (_error->PendingError() == true || Map->Size() == 0)
 692    {
 693       _error->Discard();
 694       return false;
 695    }
 696
 697    /* Now we check every index file, see if it is in the cache,
 698       verify the IMS data and check that it is on the disk too.. */
 699    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 700    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 701    for (; Start != End; Start++)
 702    {
 703       if ((*Start)->HasPackages() == false)
 704          continue;
 705
 706       if ((*Start)->Exists() == false)
 707       {
 708 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 709          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 710                           (*Start)->Describe().c_str());
 711 #endif
 712          continue;
 713       }
 714
 715       // FindInCache is also expected to do an IMS check.
 716       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 717       if (File.end() == true)
 718          return false;
 719
 720       Visited[File->ID] = true;
 721    }
 722
 723    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 724       if (Visited[I] == false)
 725          return false;
 726
 727    if (_error->PendingError() == true)
 728    {
 729       _error->Discard();
 730       return false;
 731    }
 732
 733    if (OutMap != 0)
 734       *OutMap = Map.UnGuard();
 735    return true;
 736 }
 737                                                                         /*}}}*/
 738 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 739 // ---------------------------------------------------------------------
 740 /* Size is kind of an abstract notion that is only used for the progress
 741    meter */
 742 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 743 {
 744    unsigned long TotalSize = 0;
 745    for (; Start != End; Start++)
 746    {
 747       if ((*Start)->HasPackages() == false)
 748          continue;
 749       TotalSize += (*Start)->Size();
 750    }
 751    return TotalSize;
 752 }
 753                                                                         /*}}}*/
 754 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 755 // ---------------------------------------------------------------------
 756 /* */
 757 static bool BuildCache(pkgCacheGenerator &Gen,
 758                        OpProgress &Progress,
 759                        unsigned long &CurrentSize,unsigned long TotalSize,
 760                        FileIterator Start, FileIterator End)
 761 {
 762    FileIterator I;
 763    for (I = Start; I != End; I++)
 764    {
 765       if ((*I)->HasPackages() == false)
 766          continue;
 767
 768       if ((*I)->Exists() == false)
 769          continue;
 770
 771       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 772       {
 773          _error->Warning("Duplicate sources.list entry %s",
 774                          (*I)->Describe().c_str());
 775          continue;
 776       }
 777
 778       unsigned long Size = (*I)->Size();
 779       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 780       CurrentSize += Size;
 781
 782       if ((*I)->Merge(Gen,Progress) == false)
 783          return false;
 784    }
 785
 786    if (Gen.HasFileDeps() == true)
 787    {
 788       Progress.Done();
 789       TotalSize = ComputeSize(Start, End);
 790       CurrentSize = 0;
 791       for (I = Start; I != End; I++)
 792       {
 793          unsigned long Size = (*I)->Size();
 794          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 795          CurrentSize += Size;
 796          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 797             return false;
 798       }
 799    }
 800
 801    return true;
 802 }
 803                                                                         /*}}}*/
 804 // MakeStatusCache - Construct the status cache                         /*{{{*/
 805 // ---------------------------------------------------------------------
 806 /* This makes sure that the status cache (the cache that has all
 807    index files from the sources list and all local ones) is ready
 808    to be mmaped. If OutMap is not zero then a MMap object representing
 809    the cache will be stored there. This is pretty much mandetory if you
 810    are using AllowMem. AllowMem lets the function be run as non-root
 811    where it builds the cache 'fast' into a memory buffer. */
 812 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 813                         MMap **OutMap,bool AllowMem)
 814 {
 815    unsigned long MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 816
 817    vector<pkgIndexFile *> Files;
 818    for (vector<metaIndex *>::const_iterator i = List.begin();
 819         i != List.end();
 820         i++)
 821    {
 822       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 823       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 824            j != Indexes->end();
 825            j++)
 826          Files.push_back (*j);
 827    }
 828
 829    unsigned long EndOfSource = Files.size();
 830    if (_system->AddStatusFiles(Files) == false)
 831       return false;
 832
 833    // Decide if we can write to the files..
 834    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 835    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 836
 837    // Decide if we can write to the cache
 838    bool Writeable = false;
 839    if (CacheFile.empty() == false)
 840       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 841    else
 842       if (SrcCacheFile.empty() == false)
 843          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 844
 845    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 846       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 847
 848    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 849
 850    // Cache is OK, Fin.
 851    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 852    {
 853       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 854       return true;
 855    }
 856
 857    /* At this point we know we need to reconstruct the package cache,
 858       begin. */
 859    SPtr<FileFd> CacheF;
 860    SPtr<DynamicMMap> Map;
 861    if (Writeable == true && CacheFile.empty() == false)
 862    {
 863       unlink(CacheFile.c_str());
 864       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 865       fchmod(CacheF->Fd(),0644);
 866       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 867       if (_error->PendingError() == true)
 868          return false;
 869    }
 870    else
 871    {
 872       // Just build it in memory..
 873       Map = new DynamicMMap(MMap::Public,MapSize);
 874    }
 875
 876    // Lets try the source cache.
 877    unsigned long CurrentSize = 0;
 878    unsigned long TotalSize = 0;
 879    if (CheckValidity(SrcCacheFile,Files.begin(),
 880                      Files.begin()+EndOfSource) == true)
 881    {
 882       // Preload the map with the source cache
 883       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 884       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 885                        SCacheF.Size()) == false)
 886          return false;
 887
 888       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 889
 890       // Build the status cache
 891       pkgCacheGenerator Gen(Map.Get(),&Progress);
 892       if (_error->PendingError() == true)
 893          return false;
 894       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 895                      Files.begin()+EndOfSource,Files.end()) == false)
 896          return false;
 897    }
 898    else
 899    {
 900       TotalSize = ComputeSize(Files.begin(),Files.end());
 901
 902       // Build the source cache
 903       pkgCacheGenerator Gen(Map.Get(),&Progress);
 904       if (_error->PendingError() == true)
 905          return false;
 906       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 907                      Files.begin(),Files.begin()+EndOfSource) == false)
 908          return false;
 909
 910       // Write it back
 911       if (Writeable == true && SrcCacheFile.empty() == false)
 912       {
 913          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 914          if (_error->PendingError() == true)
 915             return false;
 916
 917          fchmod(SCacheF.Fd(),0644);
 918
 919          // Write out the main data
 920          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 921             return _error->Error(_("IO Error saving source cache"));
 922          SCacheF.Sync();
 923
 924          // Write out the proper header
 925          Gen.GetCache().HeaderP->Dirty = false;
 926          if (SCacheF.Seek(0) == false ||
 927              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 928             return _error->Error(_("IO Error saving source cache"));
 929          Gen.GetCache().HeaderP->Dirty = true;
 930          SCacheF.Sync();
 931       }
 932
 933       // Build the status cache
 934       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 935                      Files.begin()+EndOfSource,Files.end()) == false)
 936          return false;
 937    }
 938
 939    if (_error->PendingError() == true)
 940       return false;
 941    if (OutMap != 0)
 942    {
 943       if (CacheF != 0)
 944       {
 945          delete Map.UnGuard();
 946          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 947       }
 948       else
 949       {
 950          *OutMap = Map.UnGuard();
 951       }
 952    }
 953
 954    return true;
 955 }
 956                                                                         /*}}}*/
 957 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 958 // ---------------------------------------------------------------------
 959 /* */
 960 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 961 {
 962    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
 963    vector<pkgIndexFile *> Files;
 964    unsigned long EndOfSource = Files.size();
 965    if (_system->AddStatusFiles(Files) == false)
 966       return false;
 967
 968    SPtr<DynamicMMap> Map;
 969    Map = new DynamicMMap(MMap::Public,MapSize);
 970    unsigned long CurrentSize = 0;
 971    unsigned long TotalSize = 0;
 972
 973    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 974
 975    // Build the status cache
 976    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 977    pkgCacheGenerator Gen(Map.Get(),&Progress);
 978    if (_error->PendingError() == true)
 979       return false;
 980    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 981                   Files.begin()+EndOfSource,Files.end()) == false)
 982       return false;
 983
 984    if (_error->PendingError() == true)
 985       return false;
 986    *OutMap = Map.UnGuard();
 987
 988    return true;
 989 }
 990                                                                         /*}}}*/