apt-pkg/pkgcachegen.cc.orig

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <vector>
  31
  32 #include <sys/stat.h>
  33 #include <unistd.h>
  34 #include <errno.h>
  35 #include <stdio.h>
  36                                                                         /*}}}*/
  37 typedef vector<pkgIndexFile *>::iterator FileIterator;
  38
  39 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* We set the diry flag and make sure that is written to the disk */
  42 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  43                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  44                     FoundFileDeps(0)
  45 {
  46    CurrentFile = 0;
  47    memset(UniqHash,0,sizeof(UniqHash));
  48
  49    if (_error->PendingError() == true)
  50       return;
  51
  52    if (Map.Size() == 0)
  53    {
  54       // Setup the map interface..
  55       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  56       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  57          return;
  58
  59       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  60
  61       // Starting header
  62       *Cache.HeaderP = pkgCache::Header();
  63       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  64       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  65       Cache.ReMap();
  66    }
  67    else
  68    {
  69       // Map directly from the existing file
  70       Cache.ReMap();
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72       if (Cache.VS != _system->VS)
  73       {
  74          _error->Error(_("Cache has an incompatible versioning system"));
  75          return;
  76       }
  77    }
  78
  79    Cache.HeaderP->Dirty = true;
  80    Map.Sync(0,sizeof(pkgCache::Header));
  81 }
  82                                                                         /*}}}*/
  83 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* We sync the data then unset the dirty flag in two steps so as to
  86    advoid a problem during a crash */
  87 pkgCacheGenerator::~pkgCacheGenerator()
  88 {
  89    if (_error->PendingError() == true)
  90       return;
  91    if (Map.Sync() == false)
  92       return;
  93
  94    Cache.HeaderP->Dirty = false;
  95    Map.Sync(0,sizeof(pkgCache::Header));
  96 }
  97                                                                         /*}}}*/
  98 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* This provides the generation of the entries in the cache. Each loop
 101    goes through a single package record from the underlying parse engine. */
 102 bool pkgCacheGenerator::MergeList(ListParser &List,
 103                                   pkgCache::VerIterator *OutVer)
 104 {
 105    List.Owner = this;
 106
 107    unsigned int Counter = 0;
 108    while (List.Step() == true)
 109    {
 110       // Get a pointer to the package structure
 111       string PackageName = List.Package();
 112       if (PackageName.empty() == true)
 113          return false;
 114
 115       pkgCache::PkgIterator Pkg;
 116       if (NewPackage(Pkg,PackageName) == false)
 117          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 118       Counter++;
 119       if (Counter % 100 == 0 && Progress != 0)
 120          Progress->Progress(List.Offset());
 121
 122       /* Get a pointer to the version structure. We know the list is sorted
 123          so we use that fact in the search. Insertion of new versions is
 124          done with correct sorting */
 125       string Version = List.Version();
 126       if (Version.empty() == true)
 127       {
 128          // we first process the package, then the descriptions
 129          // (this has the bonus that we get MMap error when we run out
 130          //  of MMap space)
 131          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 132             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 133                                  PackageName.c_str());
 134
 135          // Find the right version to write the description
 136          MD5SumValue CurMd5 = List.Description_md5();
 137          pkgCache::VerIterator Ver = Pkg.VersionList();
 138          map_ptrloc *LastVer = &Pkg->VersionList;
 139
 140          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 141          {
 142             pkgCache::DescIterator Desc = Ver.DescriptionList();
 143             map_ptrloc *LastDesc = &Ver->DescriptionList;
 144             bool duplicate=false;
 145
 146             // don't add a new description if we have one for the given
 147             // md5 && language
 148             for ( ; Desc.end() == false; Desc++)
 149                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 150                    Desc.LanguageCode() == List.DescriptionLanguage())
 151                   duplicate=true;
 152             if(duplicate)
 153                continue;
 154
 155             for (Desc = Ver.DescriptionList();
 156                  Desc.end() == false;
 157                  LastDesc = &Desc->NextDesc, Desc++)
 158             {
 159                if (MD5SumValue(Desc.md5()) == CurMd5)
 160                {
 161                   // Add new description
 162                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 163                   Desc->ParentPkg = Pkg.Index();
 164
 165                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 166                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 167                   break;
 168                }
 169             }
 170          }
 171
 172          continue;
 173       }
 174
 175       pkgCache::VerIterator Ver = Pkg.VersionList();
 176       map_ptrloc *LastVer = &Pkg->VersionList;
 177       int Res = 1;
 178       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 179       {
 180          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 181          if (Res >= 0)
 182             break;
 183       }
 184
 185       /* We already have a version for this item, record that we
 186          saw it */
 187       unsigned long Hash = List.VersionHash();
 188       if (Res == 0 && Ver->Hash == Hash)
 189       {
 190          if (List.UsePackage(Pkg,Ver) == false)
 191             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 192                                  PackageName.c_str());
 193
 194          if (NewFileVer(Ver,List) == false)
 195             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 196                                  PackageName.c_str());
 197
 198          // Read only a single record and return
 199          if (OutVer != 0)
 200          {
 201             *OutVer = Ver;
 202             FoundFileDeps |= List.HasFileDeps();
 203             return true;
 204          }
 205
 206          continue;
 207       }
 208
 209       // Skip to the end of the same version set.
 210       if (Res == 0)
 211       {
 212          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 213          {
 214             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 215             if (Res != 0)
 216                break;
 217          }
 218       }
 219
 220       // Add a new version
 221       *LastVer = NewVersion(Ver,Version,*LastVer);
 222       Ver->ParentPkg = Pkg.Index();
 223       Ver->Hash = Hash;
 224
 225       if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false)
 226          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 227                               PackageName.c_str());
 228
 229       if (List.UsePackage(Pkg,Ver) == false)
 230          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 231                               PackageName.c_str());
 232
 233       if (NewFileVer(Ver,List) == false)
 234          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 235                               PackageName.c_str());
 236
 237       // Read only a single record and return
 238       if (OutVer != 0)
 239       {
 240          *OutVer = Ver;
 241          FoundFileDeps |= List.HasFileDeps();
 242          return true;
 243       }
 244
 245       /* Record the Description data. Description data always exist in
 246          Packages and Translation-* files. */
 247       pkgCache::DescIterator Desc = Ver.DescriptionList();
 248       map_ptrloc *LastDesc = &Ver->DescriptionList;
 249
 250       // Skip to the end of description set
 251       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 252
 253       // Add new description
 254       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 255       Desc->ParentPkg = Pkg.Index();
 256
 257       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 258          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 259    }
 260
 261    FoundFileDeps |= List.HasFileDeps();
 262
 263    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 264       return _error->Error(_("Wow, you exceeded the number of package "
 265                              "names this APT is capable of."));
 266    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 267       return _error->Error(_("Wow, you exceeded the number of versions "
 268                              "this APT is capable of."));
 269    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 270       return _error->Error(_("Wow, you exceeded the number of descriptions "
 271                              "this APT is capable of."));
 272    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 273       return _error->Error(_("Wow, you exceeded the number of dependencies "
 274                              "this APT is capable of."));
 275    return true;
 276 }
 277                                                                         /*}}}*/
 278 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 279 // ---------------------------------------------------------------------
 280 /* If we found any file depends while parsing the main list we need to
 281    resolve them. Since it is undesired to load the entire list of files
 282    into the cache as virtual packages we do a two stage effort. MergeList
 283    identifies the file depends and this creates Provdies for them by
 284    re-parsing all the indexs. */
 285 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 286 {
 287    List.Owner = this;
 288
 289    unsigned int Counter = 0;
 290    while (List.Step() == true)
 291    {
 292       string PackageName = List.Package();
 293       if (PackageName.empty() == true)
 294          return false;
 295       string Version = List.Version();
 296       if (Version.empty() == true)
 297          continue;
 298
 299       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 300       if (Pkg.end() == true)
 301          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 302                                 PackageName.c_str());
 303       Counter++;
 304       if (Counter % 100 == 0 && Progress != 0)
 305          Progress->Progress(List.Offset());
 306
 307       unsigned long Hash = List.VersionHash();
 308       pkgCache::VerIterator Ver = Pkg.VersionList();
 309       for (; Ver.end() == false; Ver++)
 310       {
 311          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 312          {
 313             if (List.CollectFileProvides(Cache,Ver) == false)
 314                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 315             break;
 316          }
 317       }
 318
 319       if (Ver.end() == true)
 320          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 321    }
 322
 323    return true;
 324 }
 325                                                                         /*}}}*/
 326 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 327 // ---------------------------------------------------------------------
 328 /* This creates a new package structure and adds it to the hash table */
 329 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 330 {
 331    Pkg = Cache.FindPkg(Name);
 332    if (Pkg.end() == false)
 333       return true;
 334
 335    // Get a structure
 336    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 337    if (Package == 0)
 338       return false;
 339
 340    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 341
 342    // Insert it into the hash table
 343    unsigned long Hash = Cache.Hash(Name);
 344    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 345    Cache.HeaderP->HashTable[Hash] = Package;
 346
 347    // Set the name and the ID
 348    Pkg->Name = Map.WriteString(Name);
 349    if (Pkg->Name == 0)
 350       return false;
 351    Pkg->ID = Cache.HeaderP->PackageCount++;
 352
 353    return true;
 354 }
 355                                                                         /*}}}*/
 356 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 357 // ---------------------------------------------------------------------
 358 /* */
 359 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 360                                    ListParser &List)
 361 {
 362    if (CurrentFile == 0)
 363       return true;
 364
 365    // Get a structure
 366    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 367    if (VerFile == 0)
 368       return 0;
 369
 370    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 371    VF->File = CurrentFile - Cache.PkgFileP;
 372
 373    // Link it to the end of the list
 374    map_ptrloc *Last = &Ver->FileList;
 375    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 376       Last = &V->NextFile;
 377    VF->NextFile = *Last;
 378    *Last = VF.Index();
 379
 380    VF->Offset = List.Offset();
 381    VF->Size = List.Size();
 382    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 383       Cache.HeaderP->MaxVerFileSize = VF->Size;
 384    Cache.HeaderP->VerFileCount++;
 385
 386    return true;
 387 }
 388                                                                         /*}}}*/
 389 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* This puts a version structure in the linked list */
 392 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 393                                             const string &VerStr,
 394                                             unsigned long Next)
 395 {
 396    // Get a structure
 397    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 398    if (Version == 0)
 399       return 0;
 400
 401    // Fill it in
 402    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 403    Ver->NextVer = Next;
 404    Ver->ID = Cache.HeaderP->VersionCount++;
 405    Ver->VerStr = Map.WriteString(VerStr);
 406    if (Ver->VerStr == 0)
 407       return 0;
 408
 409    return Version;
 410 }
 411                                                                         /*}}}*/
 412 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 413 // ---------------------------------------------------------------------
 414 /* */
 415 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 416                                    ListParser &List)
 417 {
 418    if (CurrentFile == 0)
 419       return true;
 420
 421    // Get a structure
 422    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 423    if (DescFile == 0)
 424       return false;
 425
 426    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 427    DF->File = CurrentFile - Cache.PkgFileP;
 428
 429    // Link it to the end of the list
 430    map_ptrloc *Last = &Desc->FileList;
 431    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 432       Last = &D->NextFile;
 433
 434    DF->NextFile = *Last;
 435    *Last = DF.Index();
 436
 437    DF->Offset = List.Offset();
 438    DF->Size = List.Size();
 439    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 440       Cache.HeaderP->MaxDescFileSize = DF->Size;
 441    Cache.HeaderP->DescFileCount++;
 442
 443    return true;
 444 }
 445                                                                         /*}}}*/
 446 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* This puts a description structure in the linked list */
 449 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 450                                             const string &Lang,
 451                                             const MD5SumValue &md5sum,
 452                                             map_ptrloc Next)
 453 {
 454    // Get a structure
 455    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 456    if (Description == 0)
 457       return 0;
 458
 459    // Fill it in
 460    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 461    Desc->NextDesc = Next;
 462    Desc->ID = Cache.HeaderP->DescriptionCount++;
 463    Desc->language_code = Map.WriteString(Lang);
 464    Desc->md5sum = Map.WriteString(md5sum.Value());
 465    if (Desc->language_code == 0 || Desc->md5sum == 0)
 466       return 0;
 467
 468    return Description;
 469 }
 470                                                                         /*}}}*/
 471 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 472 // ---------------------------------------------------------------------
 473 /* This creates a dependency element in the tree. It is linked to the
 474    version and to the package that it is pointing to. */
 475 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 476                                                const string &PackageName,
 477                                                const string &Version,
 478                                                unsigned int Op,
 479                                                unsigned int Type)
 480 {
 481    pkgCache &Cache = Owner->Cache;
 482
 483    // Get a structure
 484    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 485    if (Dependency == 0)
 486       return false;
 487
 488    // Fill it in
 489    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 490    Dep->ParentVer = Ver.Index();
 491    Dep->Type = Type;
 492    Dep->CompareOp = Op;
 493    Dep->ID = Cache.HeaderP->DependsCount++;
 494
 495    // Locate the target package
 496    pkgCache::PkgIterator Pkg;
 497    if (Owner->NewPackage(Pkg,PackageName) == false)
 498       return false;
 499
 500    // Probe the reverse dependency list for a version string that matches
 501    if (Version.empty() == false)
 502    {
 503 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 504          if (I->Version != 0 && I.TargetVer() == Version)
 505             Dep->Version = I->Version;*/
 506       if (Dep->Version == 0)
 507          if ((Dep->Version = WriteString(Version)) == 0)
 508             return false;
 509    }
 510
 511    // Link it to the package
 512    Dep->Package = Pkg.Index();
 513    Dep->NextRevDepends = Pkg->RevDepends;
 514    Pkg->RevDepends = Dep.Index();
 515
 516    /* Link it to the version (at the end of the list)
 517       Caching the old end point speeds up generation substantially */
 518    if (OldDepVer != Ver)
 519    {
 520       OldDepLast = &Ver->DependsList;
 521       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 522          OldDepLast = &D->NextDepends;
 523       OldDepVer = Ver;
 524    }
 525
 526    // Is it a file dependency?
 527    if (PackageName[0] == '/')
 528       FoundFileDeps = true;
 529
 530    Dep->NextDepends = *OldDepLast;
 531    *OldDepLast = Dep.Index();
 532    OldDepLast = &Dep->NextDepends;
 533
 534    return true;
 535 }
 536                                                                         /*}}}*/
 537 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 538 // ---------------------------------------------------------------------
 539 /* */
 540 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 541                                                 const string &PackageName,
 542                                                 const string &Version)
 543 {
 544    pkgCache &Cache = Owner->Cache;
 545
 546    // We do not add self referencing provides
 547    if (Ver.ParentPkg().Name() == PackageName)
 548       return true;
 549
 550    // Get a structure
 551    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 552    if (Provides == 0)
 553       return false;
 554    Cache.HeaderP->ProvidesCount++;
 555
 556    // Fill it in
 557    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 558    Prv->Version = Ver.Index();
 559    Prv->NextPkgProv = Ver->ProvidesList;
 560    Ver->ProvidesList = Prv.Index();
 561    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 562       return false;
 563
 564    // Locate the target package
 565    pkgCache::PkgIterator Pkg;
 566    if (Owner->NewPackage(Pkg,PackageName) == false)
 567       return false;
 568
 569    // Link it to the package
 570    Prv->ParentPkg = Pkg.Index();
 571    Prv->NextProvides = Pkg->ProvidesList;
 572    Pkg->ProvidesList = Prv.Index();
 573
 574    return true;
 575 }
 576                                                                         /*}}}*/
 577 // ListParser::NewTag - Create a Tag element                            /*{{{*/
 578 // ---------------------------------------------------------------------
 579 /* */
 580 bool pkgCacheGenerator::ListParser::NewTag(pkgCache::PkgIterator Pkg,
 581                                            const char *NameStart,
 582                                            unsigned int NameSize)
 583 {
 584    pkgCache &Cache = Owner->Cache;
 585
 586    // Get a structure
 587    unsigned long Tagg = Owner->Map.Allocate(sizeof(pkgCache::Tag));
 588    if (Tagg == 0)
 589       return false;
 590    Cache.HeaderP->TagCount++;
 591
 592    // Fill it in
 593    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
 594    Tg->Name = WriteString(NameStart,NameSize);
 595    if (Tg->Name == 0)
 596       return false;
 597    Tg->NextTag = Pkg->TagList;
 598    Pkg->TagList = Tg.Index();
 599
 600    return true;
 601 }
 602                                                                         /*}}}*/
 603 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 604 // ---------------------------------------------------------------------
 605 /* This is used to select which file is to be associated with all newly
 606    added versions. The caller is responsible for setting the IMS fields. */
 607 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 608                                    const pkgIndexFile &Index,
 609                                    unsigned long Flags)
 610 {
 611    // Get some space for the structure
 612    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 613    if (CurrentFile == Cache.PkgFileP)
 614       return false;
 615
 616    // Fill it in
 617    CurrentFile->FileName = Map.WriteString(File);
 618    CurrentFile->Site = WriteUniqString(Site);
 619    CurrentFile->NextFile = Cache.HeaderP->FileList;
 620    CurrentFile->Flags = Flags;
 621    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 622    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 623    PkgFileName = File;
 624    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 625    Cache.HeaderP->PackageFileCount++;
 626
 627    if (CurrentFile->FileName == 0)
 628       return false;
 629
 630    if (Progress != 0)
 631       Progress->SubProgress(Index.Size());
 632    return true;
 633 }
 634                                                                         /*}}}*/
 635 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 636 // ---------------------------------------------------------------------
 637 /* This is used to create handles to strings. Given the same text it
 638    always returns the same number */
 639 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 640                                                  unsigned int Size)
 641 {
 642    /* We use a very small transient hash table here, this speeds up generation
 643       by a fair amount on slower machines */
 644    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 645    if (Bucket != 0 &&
 646        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 647       return Bucket->String;
 648
 649    // Search for an insertion point
 650    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 651    int Res = 1;
 652    map_ptrloc *Last = &Cache.HeaderP->StringList;
 653    for (; I != Cache.StringItemP; Last = &I->NextItem,
 654         I = Cache.StringItemP + I->NextItem)
 655    {
 656       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 657       if (Res >= 0)
 658          break;
 659    }
 660
 661    // Match
 662    if (Res == 0)
 663    {
 664       Bucket = I;
 665       return I->String;
 666    }
 667
 668    // Get a structure
 669    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 670    if (Item == 0)
 671       return 0;
 672
 673    // Fill in the structure
 674    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 675    ItemP->NextItem = I - Cache.StringItemP;
 676    *Last = Item;
 677    ItemP->String = Map.WriteString(S,Size);
 678    if (ItemP->String == 0)
 679       return 0;
 680
 681    Bucket = ItemP;
 682    return ItemP->String;
 683 }
 684                                                                         /*}}}*/
 685 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 686 // ---------------------------------------------------------------------
 687 /* This just verifies that each file in the list of index files exists,
 688    has matching attributes with the cache and the cache does not have
 689    any extra files. */
 690 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 691                           FileIterator End,MMap **OutMap = 0)
 692 {
 693    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 694    // No file, certainly invalid
 695    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 696    {
 697       if (Debug == true)
 698          std::clog << "CacheFile doesn't exist" << std::endl;
 699       return false;
 700    }
 701
 702    // Map it
 703    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 704    SPtr<MMap> Map = new MMap(CacheF,0);
 705    pkgCache Cache(Map);
 706    if (_error->PendingError() == true || Map->Size() == 0)
 707    {
 708       if (Debug == true)
 709          std::clog << "Errors are pending or Map is empty()" << std::endl;
 710       _error->Discard();
 711       return false;
 712    }
 713
 714    /* Now we check every index file, see if it is in the cache,
 715       verify the IMS data and check that it is on the disk too.. */
 716    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 717    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 718    for (; Start != End; Start++)
 719    {
 720       if (Debug == true)
 721          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 722       if ((*Start)->HasPackages() == false)
 723       {
 724          if (Debug == true)
 725             std::clog << "Has NO packages" << std::endl;
 726          continue;
 727       }
 728
 729       if ((*Start)->Exists() == false)
 730       {
 731 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 732          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 733                           (*Start)->Describe().c_str());
 734 #endif
 735          if (Debug == true)
 736             std::clog << "file doesn't exist" << std::endl;
 737          continue;
 738       }
 739
 740       // FindInCache is also expected to do an IMS check.
 741       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 742       if (File.end() == true)
 743       {
 744          if (Debug == true)
 745             std::clog << "FindInCache returned end-Pointer" << std::endl;
 746          return false;
 747       }
 748
 749       Visited[File->ID] = true;
 750       if (Debug == true)
 751          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 752    }
 753
 754    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 755       if (Visited[I] == false)
 756       {
 757          if (Debug == true)
 758             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 759          return false;
 760       }
 761
 762    if (_error->PendingError() == true)
 763    {
 764       if (Debug == true)
 765       {
 766          std::clog << "Validity failed because of pending errors:" << std::endl;
 767          _error->DumpErrors();
 768       }
 769       _error->Discard();
 770       return false;
 771    }
 772
 773    if (OutMap != 0)
 774       *OutMap = Map.UnGuard();
 775    return true;
 776 }
 777                                                                         /*}}}*/
 778 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 779 // ---------------------------------------------------------------------
 780 /* Size is kind of an abstract notion that is only used for the progress
 781    meter */
 782 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 783 {
 784    unsigned long TotalSize = 0;
 785    for (; Start != End; Start++)
 786    {
 787       if ((*Start)->HasPackages() == false)
 788          continue;
 789       TotalSize += (*Start)->Size();
 790    }
 791    return TotalSize;
 792 }
 793                                                                         /*}}}*/
 794 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 795 // ---------------------------------------------------------------------
 796 /* */
 797 static bool BuildCache(pkgCacheGenerator &Gen,
 798                        OpProgress &Progress,
 799                        unsigned long &CurrentSize,unsigned long TotalSize,
 800                        FileIterator Start, FileIterator End)
 801 {
 802    FileIterator I;
 803    for (I = Start; I != End; I++)
 804    {
 805       if ((*I)->HasPackages() == false)
 806          continue;
 807
 808       if ((*I)->Exists() == false)
 809          continue;
 810
 811       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 812       {
 813          _error->Warning("Duplicate sources.list entry %s",
 814                          (*I)->Describe().c_str());
 815          continue;
 816       }
 817
 818       unsigned long Size = (*I)->Size();
 819       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 820       CurrentSize += Size;
 821
 822       if ((*I)->Merge(Gen,Progress) == false)
 823          return false;
 824    }
 825
 826    if (Gen.HasFileDeps() == true)
 827    {
 828       Progress.Done();
 829       TotalSize = ComputeSize(Start, End);
 830       CurrentSize = 0;
 831       for (I = Start; I != End; I++)
 832       {
 833          unsigned long Size = (*I)->Size();
 834          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 835          CurrentSize += Size;
 836          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 837             return false;
 838       }
 839    }
 840
 841    return true;
 842 }
 843                                                                         /*}}}*/
 844 // MakeStatusCache - Construct the status cache                         /*{{{*/
 845 // ---------------------------------------------------------------------
 846 /* This makes sure that the status cache (the cache that has all
 847    index files from the sources list and all local ones) is ready
 848    to be mmaped. If OutMap is not zero then a MMap object representing
 849    the cache will be stored there. This is pretty much mandetory if you
 850    are using AllowMem. AllowMem lets the function be run as non-root
 851    where it builds the cache 'fast' into a memory buffer. */
 852 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 853                         MMap **OutMap,bool AllowMem)
 854 {
 855    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 856    unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 857
 858    vector<pkgIndexFile *> Files;
 859    for (vector<metaIndex *>::const_iterator i = List.begin();
 860         i != List.end();
 861         i++)
 862    {
 863       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 864       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 865            j != Indexes->end();
 866            j++)
 867          Files.push_back (*j);
 868    }
 869
 870    unsigned long const EndOfSource = Files.size();
 871    if (_system->AddStatusFiles(Files) == false)
 872       return false;
 873
 874    // Decide if we can write to the files..
 875    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 876    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 877
 878    // Decide if we can write to the cache
 879    bool Writeable = false;
 880    if (CacheFile.empty() == false)
 881       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 882    else
 883       if (SrcCacheFile.empty() == false)
 884          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 885    if (Debug == true)
 886       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
 887
 888    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 889       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 890
 891    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 892
 893    // Cache is OK, Fin.
 894    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 895    {
 896       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 897       if (Debug == true)
 898          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
 899       return true;
 900    }
 901    else if (Debug == true)
 902          std::clog << "pkgcache.bin is NOT valid" << std::endl;
 903
 904    /* At this point we know we need to reconstruct the package cache,
 905       begin. */
 906    SPtr<FileFd> CacheF;
 907    SPtr<DynamicMMap> Map;
 908    if (Writeable == true && CacheFile.empty() == false)
 909    {
 910       unlink(CacheFile.c_str());
 911       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 912       fchmod(CacheF->Fd(),0644);
 913       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 914       if (_error->PendingError() == true)
 915          return false;
 916       if (Debug == true)
 917          std::clog << "Open filebased MMap" << std::endl;
 918    }
 919    else
 920    {
 921       // Just build it in memory..
 922       Map = new DynamicMMap(0,MapSize);
 923       if (Debug == true)
 924          std::clog << "Open memory Map (not filebased)" << std::endl;
 925    }
 926
 927    // Lets try the source cache.
 928    unsigned long CurrentSize = 0;
 929    unsigned long TotalSize = 0;
 930    if (CheckValidity(SrcCacheFile,Files.begin(),
 931                      Files.begin()+EndOfSource) == true)
 932    {
 933       if (Debug == true)
 934          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
 935       // Preload the map with the source cache
 936       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 937       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
 938       if ((alloc == 0 && _error->PendingError())
 939                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
 940                                 SCacheF.Size()) == false)
 941          return false;
 942
 943       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 944
 945       // Build the status cache
 946       pkgCacheGenerator Gen(Map.Get(),&Progress);
 947       if (_error->PendingError() == true)
 948          return false;
 949       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 950                      Files.begin()+EndOfSource,Files.end()) == false)
 951          return false;
 952    }
 953    else
 954    {
 955       if (Debug == true)
 956          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
 957       TotalSize = ComputeSize(Files.begin(),Files.end());
 958
 959       // Build the source cache
 960       pkgCacheGenerator Gen(Map.Get(),&Progress);
 961       if (_error->PendingError() == true)
 962          return false;
 963       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 964                      Files.begin(),Files.begin()+EndOfSource) == false)
 965          return false;
 966
 967       // Write it back
 968       if (Writeable == true && SrcCacheFile.empty() == false)
 969       {
 970          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 971          if (_error->PendingError() == true)
 972             return false;
 973
 974          fchmod(SCacheF.Fd(),0644);
 975
 976          // Write out the main data
 977          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 978             return _error->Error(_("IO Error saving source cache"));
 979          SCacheF.Sync();
 980
 981          // Write out the proper header
 982          Gen.GetCache().HeaderP->Dirty = false;
 983          if (SCacheF.Seek(0) == false ||
 984              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 985             return _error->Error(_("IO Error saving source cache"));
 986          Gen.GetCache().HeaderP->Dirty = true;
 987          SCacheF.Sync();
 988       }
 989
 990       // Build the status cache
 991       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 992                      Files.begin()+EndOfSource,Files.end()) == false)
 993          return false;
 994    }
 995    if (Debug == true)
 996       std::clog << "Caches are ready for shipping" << std::endl;
 997
 998    if (_error->PendingError() == true)
 999       return false;
1000    if (OutMap != 0)
1001    {
1002       if (CacheF != 0)
1003       {
1004          delete Map.UnGuard();
1005          *OutMap = new MMap(*CacheF,0);
1006       }
1007       else
1008       {
1009          *OutMap = Map.UnGuard();
1010       }
1011    }
1012
1013    return true;
1014 }
1015                                                                         /*}}}*/
1016 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
1017 // ---------------------------------------------------------------------
1018 /* */
1019 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1020 {
1021    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
1022    vector<pkgIndexFile *> Files;
1023    unsigned long EndOfSource = Files.size();
1024    if (_system->AddStatusFiles(Files) == false)
1025       return false;
1026
1027    SPtr<DynamicMMap> Map = new DynamicMMap(0,MapSize);
1028    unsigned long CurrentSize = 0;
1029    unsigned long TotalSize = 0;
1030
1031    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1032
1033    // Build the status cache
1034    Progress.OverallProgress(0,1,1,_("Reading package lists"));
1035    pkgCacheGenerator Gen(Map.Get(),&Progress);
1036    if (_error->PendingError() == true)
1037       return false;
1038    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1039                   Files.begin()+EndOfSource,Files.end()) == false)
1040       return false;
1041
1042    if (_error->PendingError() == true)
1043       return false;
1044    *OutMap = Map.UnGuard();
1045
1046    return true;
1047 }
1048                                                                         /*}}}*/