]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
'apt-get update' no longer does 'Building Dependency Tree'.
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7a3c2ab0 3// $Id: pkgcachegen.cc,v 1.51 2002/04/26 05:35:57 jgg Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
6c139d6e 13#ifdef __GNUG__
094a497d 14#pragma implementation "apt-pkg/pkgcachegen.h"
6c139d6e
AL
15#endif
16
b2e465d6
AL
17#define APT_COMPATIBILITY 986
18
094a497d
AL
19#include <apt-pkg/pkgcachegen.h>
20#include <apt-pkg/error.h>
21#include <apt-pkg/version.h>
b35d2f5f
AL
22#include <apt-pkg/progress.h>
23#include <apt-pkg/sourcelist.h>
24#include <apt-pkg/configuration.h>
cdcc6d34 25#include <apt-pkg/strutl.h>
b2e465d6
AL
26#include <apt-pkg/sptr.h>
27#include <apt-pkg/pkgsystem.h>
578bfd0a 28
b2e465d6 29#include <apti18n.h>
e7b470ee
AL
30
31#include <vector>
32
578bfd0a
AL
33#include <sys/stat.h>
34#include <unistd.h>
803fafcb 35#include <errno.h>
7ef72446 36#include <stdio.h>
1ae93c94 37#include <system.h>
578bfd0a 38 /*}}}*/
e7b470ee 39typedef vector<pkgIndexFile *>::iterator FileIterator;
578bfd0a
AL
40
41// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
42// ---------------------------------------------------------------------
43/* We set the diry flag and make sure that is written to the disk */
b2e465d6
AL
44pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45 Map(*pMap), Cache(pMap,false), Progress(Prog)
578bfd0a 46{
ddc1d8d0 47 CurrentFile = 0;
b2e465d6 48 memset(UniqHash,0,sizeof(UniqHash));
ddc1d8d0 49
578bfd0a
AL
50 if (_error->PendingError() == true)
51 return;
b2e465d6 52
578bfd0a
AL
53 if (Map.Size() == 0)
54 {
b2e465d6
AL
55 // Setup the map interface..
56 Cache.HeaderP = (pkgCache::Header *)Map.Data();
578bfd0a 57 Map.RawAllocate(sizeof(pkgCache::Header));
b2e465d6
AL
58 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
59
60 // Starting header
578bfd0a 61 *Cache.HeaderP = pkgCache::Header();
b2e465d6
AL
62 Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
63 Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
64 Cache.ReMap();
578bfd0a 65 }
b2e465d6
AL
66 else
67 {
68 // Map directly from the existing file
69 Cache.ReMap();
70 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
71 if (Cache.VS != _system->VS)
72 {
73 _error->Error(_("Cache has an incompatible versioning system"));
74 return;
75 }
76 }
77
578bfd0a
AL
78 Cache.HeaderP->Dirty = true;
79 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
80}
81 /*}}}*/
82// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
83// ---------------------------------------------------------------------
84/* We sync the data then unset the dirty flag in two steps so as to
85 advoid a problem during a crash */
86pkgCacheGenerator::~pkgCacheGenerator()
87{
88 if (_error->PendingError() == true)
89 return;
90 if (Map.Sync() == false)
91 return;
92
93 Cache.HeaderP->Dirty = false;
94 Map.Sync(0,sizeof(pkgCache::Header));
95}
96 /*}}}*/
97// CacheGenerator::MergeList - Merge the package list /*{{{*/
98// ---------------------------------------------------------------------
99/* This provides the generation of the entries in the cache. Each loop
100 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
101bool pkgCacheGenerator::MergeList(ListParser &List,
102 pkgCache::VerIterator *OutVer)
578bfd0a
AL
103{
104 List.Owner = this;
0149949b 105
f9eec0e7 106 unsigned int Counter = 0;
0149949b 107 while (List.Step() == true)
578bfd0a
AL
108 {
109 // Get a pointer to the package structure
9ddf7030 110 string PackageName = List.Package();
65a1e968
AL
111 if (PackageName.empty() == true)
112 return false;
113
9ddf7030 114 pkgCache::PkgIterator Pkg;
8efa2a3b 115 if (NewPackage(Pkg,PackageName) == false)
b2e465d6 116 return _error->Error(_("Error occured while processing %s (NewPackage)"),PackageName.c_str());
a246f2dc 117 Counter++;
ddc1d8d0
AL
118 if (Counter % 100 == 0 && Progress != 0)
119 Progress->Progress(List.Offset());
8ce4327b 120
578bfd0a
AL
121 /* Get a pointer to the version structure. We know the list is sorted
122 so we use that fact in the search. Insertion of new versions is
123 done with correct sorting */
124 string Version = List.Version();
f55a958f
AL
125 if (Version.empty() == true)
126 {
127 if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
7a3c2ab0
AL
128 return _error->Error(_("Error occured while processing %s (UsePackage1)"),
129 PackageName.c_str());
f55a958f
AL
130 continue;
131 }
132
578bfd0a 133 pkgCache::VerIterator Ver = Pkg.VersionList();
349cd3b8 134 map_ptrloc *Last = &Pkg->VersionList;
2246928b 135 int Res = 1;
f55a958f 136 for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
578bfd0a 137 {
c24972cb 138 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
578bfd0a
AL
139 if (Res >= 0)
140 break;
141 }
142
143 /* We already have a version for this item, record that we
144 saw it */
204fbdcc
AL
145 unsigned long Hash = List.VersionHash();
146 if (Res == 0 && Ver->Hash == Hash)
578bfd0a 147 {
f55a958f 148 if (List.UsePackage(Pkg,Ver) == false)
7a3c2ab0
AL
149 return _error->Error(_("Error occured while processing %s (UsePackage2)"),
150 PackageName.c_str());
f78439bf 151
578bfd0a 152 if (NewFileVer(Ver,List) == false)
7a3c2ab0
AL
153 return _error->Error(_("Error occured while processing %s (NewFileVer1)"),
154 PackageName.c_str());
578bfd0a 155
ddc1d8d0
AL
156 // Read only a single record and return
157 if (OutVer != 0)
158 {
159 *OutVer = Ver;
160 return true;
161 }
162
578bfd0a
AL
163 continue;
164 }
165
204fbdcc
AL
166 // Skip to the end of the same version set.
167 if (Res == 0)
168 {
169 for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
170 {
c24972cb 171 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
204fbdcc
AL
172 if (Res != 0)
173 break;
174 }
175 }
176
578bfd0a 177 // Add a new version
f55a958f
AL
178 *Last = NewVersion(Ver,Version,*Last);
179 Ver->ParentPkg = Pkg.Index();
204fbdcc 180 Ver->Hash = Hash;
578bfd0a 181 if (List.NewVersion(Ver) == false)
7a3c2ab0
AL
182 return _error->Error(_("Error occured while processing %s (NewVersion1)"),
183 PackageName.c_str());
0149949b 184
f55a958f 185 if (List.UsePackage(Pkg,Ver) == false)
7a3c2ab0
AL
186 return _error->Error(_("Error occured while processing %s (UsePackage3)"),
187 PackageName.c_str());
f55a958f 188
578bfd0a 189 if (NewFileVer(Ver,List) == false)
7a3c2ab0
AL
190 return _error->Error(_("Error occured while processing %s (NewVersion2)"),
191 PackageName.c_str());
ddc1d8d0
AL
192
193 // Read only a single record and return
194 if (OutVer != 0)
195 {
196 *OutVer = Ver;
197 return true;
198 }
578bfd0a 199 }
0149949b 200
6a3da7a6
AL
201 if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
202 return _error->Error(_("Wow, you exceeded the number of package "
203 "names this APT is capable of."));
204 if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
205 return _error->Error(_("Wow, you exceeded the number of versions "
206 "this APT is capable of."));
207 if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
208 return _error->Error(_("Wow, you exceeded the number of dependencies "
209 "this APT is capable of."));
578bfd0a
AL
210 return true;
211}
212 /*}}}*/
213// CacheGenerator::NewPackage - Add a new package /*{{{*/
214// ---------------------------------------------------------------------
215/* This creates a new package structure and adds it to the hash table */
f55a958f 216bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
578bfd0a 217{
8efa2a3b
AL
218 Pkg = Cache.FindPkg(Name);
219 if (Pkg.end() == false)
220 return true;
221
578bfd0a
AL
222 // Get a structure
223 unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
224 if (Package == 0)
225 return false;
226
f55a958f 227 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
578bfd0a
AL
228
229 // Insert it into the hash table
f55a958f 230 unsigned long Hash = Cache.Hash(Name);
578bfd0a
AL
231 Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
232 Cache.HeaderP->HashTable[Hash] = Package;
233
234 // Set the name and the ID
235 Pkg->Name = Map.WriteString(Name);
236 if (Pkg->Name == 0)
237 return false;
238 Pkg->ID = Cache.HeaderP->PackageCount++;
239
240 return true;
241}
242 /*}}}*/
243// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
244// ---------------------------------------------------------------------
245/* */
f55a958f 246bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
247 ListParser &List)
248{
ddc1d8d0
AL
249 if (CurrentFile == 0)
250 return true;
251
dcb79bae
AL
252 // Get a structure
253 unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
254 if (VerFile == 0)
255 return 0;
256
257 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
258 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
259
260 // Link it to the end of the list
349cd3b8 261 map_ptrloc *Last = &Ver->FileList;
03e39e59
AL
262 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
263 Last = &V->NextFile;
264 VF->NextFile = *Last;
265 *Last = VF.Index();
266
dcb79bae
AL
267 VF->Offset = List.Offset();
268 VF->Size = List.Size();
ad00ae81
AL
269 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
270 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
271 Cache.HeaderP->VerFileCount++;
272
f55a958f 273 return true;
578bfd0a
AL
274}
275 /*}}}*/
276// CacheGenerator::NewVersion - Create a new Version /*{{{*/
277// ---------------------------------------------------------------------
f55a958f 278/* This puts a version structure in the linked list */
578bfd0a 279unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
f55a958f 280 string VerStr,
578bfd0a
AL
281 unsigned long Next)
282{
f55a958f
AL
283 // Get a structure
284 unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
285 if (Version == 0)
0149949b 286 return 0;
f55a958f
AL
287
288 // Fill it in
289 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
f55a958f
AL
290 Ver->NextVer = Next;
291 Ver->ID = Cache.HeaderP->VersionCount++;
292 Ver->VerStr = Map.WriteString(VerStr);
293 if (Ver->VerStr == 0)
0149949b 294 return 0;
f55a958f 295
0149949b 296 return Version;
578bfd0a
AL
297}
298 /*}}}*/
dcb79bae
AL
299// ListParser::NewDepends - Create a dependency element /*{{{*/
300// ---------------------------------------------------------------------
301/* This creates a dependency element in the tree. It is linked to the
302 version and to the package that it is pointing to. */
303bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
9ddf7030
AL
304 string PackageName,
305 string Version,
dcb79bae
AL
306 unsigned int Op,
307 unsigned int Type)
308{
309 pkgCache &Cache = Owner->Cache;
310
311 // Get a structure
312 unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
313 if (Dependency == 0)
314 return false;
315
316 // Fill it in
317 pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
318 Dep->ParentVer = Ver.Index();
319 Dep->Type = Type;
320 Dep->CompareOp = Op;
321 Dep->ID = Cache.HeaderP->DependsCount++;
322
323 // Locate the target package
8efa2a3b
AL
324 pkgCache::PkgIterator Pkg;
325 if (Owner->NewPackage(Pkg,PackageName) == false)
326 return false;
dcb79bae
AL
327
328 // Probe the reverse dependency list for a version string that matches
329 if (Version.empty() == false)
330 {
b2e465d6 331/* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
dcb79bae 332 if (I->Version != 0 && I.TargetVer() == Version)
f9eec0e7 333 Dep->Version = I->Version;*/
dcb79bae
AL
334 if (Dep->Version == 0)
335 if ((Dep->Version = WriteString(Version)) == 0)
336 return false;
337 }
c1a22377 338
dcb79bae
AL
339 // Link it to the package
340 Dep->Package = Pkg.Index();
341 Dep->NextRevDepends = Pkg->RevDepends;
342 Pkg->RevDepends = Dep.Index();
343
c1a22377
AL
344 /* Link it to the version (at the end of the list)
345 Caching the old end point speeds up generation substantially */
f9eec0e7 346 if (OldDepVer != Ver)
c1a22377 347 {
f9eec0e7 348 OldDepLast = &Ver->DependsList;
c1a22377 349 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
f9eec0e7
AL
350 OldDepLast = &D->NextDepends;
351 OldDepVer = Ver;
c1a22377 352 }
dcb79bae 353
f9eec0e7
AL
354 Dep->NextDepends = *OldDepLast;
355 *OldDepLast = Dep.Index();
356 OldDepLast = &Dep->NextDepends;
c1a22377 357
dcb79bae
AL
358 return true;
359}
360 /*}}}*/
361// ListParser::NewProvides - Create a Provides element /*{{{*/
362// ---------------------------------------------------------------------
363/* */
364bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
8efa2a3b 365 string PackageName,
9ddf7030 366 string Version)
dcb79bae
AL
367{
368 pkgCache &Cache = Owner->Cache;
8efa2a3b
AL
369
370 // We do not add self referencing provides
371 if (Ver.ParentPkg().Name() == PackageName)
372 return true;
dcb79bae
AL
373
374 // Get a structure
375 unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
376 if (Provides == 0)
377 return false;
a7e66b17 378 Cache.HeaderP->ProvidesCount++;
dcb79bae
AL
379
380 // Fill it in
381 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
382 Prv->Version = Ver.Index();
383 Prv->NextPkgProv = Ver->ProvidesList;
384 Ver->ProvidesList = Prv.Index();
b2e465d6 385 if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
dcb79bae
AL
386 return false;
387
388 // Locate the target package
8efa2a3b
AL
389 pkgCache::PkgIterator Pkg;
390 if (Owner->NewPackage(Pkg,PackageName) == false)
391 return false;
dcb79bae
AL
392
393 // Link it to the package
394 Prv->ParentPkg = Pkg.Index();
395 Prv->NextProvides = Pkg->ProvidesList;
396 Pkg->ProvidesList = Prv.Index();
397
398 return true;
399}
400 /*}}}*/
578bfd0a
AL
401// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
402// ---------------------------------------------------------------------
403/* This is used to select which file is to be associated with all newly
b2e465d6
AL
404 added versions. The caller is responsible for setting the IMS fields. */
405bool pkgCacheGenerator::SelectFile(string File,string Site,
406 const pkgIndexFile &Index,
407 unsigned long Flags)
578bfd0a 408{
578bfd0a
AL
409 // Get some space for the structure
410 CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
411 if (CurrentFile == Cache.PkgFileP)
412 return false;
413
414 // Fill it in
415 CurrentFile->FileName = Map.WriteString(File);
b2e465d6 416 CurrentFile->Site = WriteUniqString(Site);
578bfd0a
AL
417 CurrentFile->NextFile = Cache.HeaderP->FileList;
418 CurrentFile->Flags = Flags;
e1b74f61 419 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
b2e465d6 420 CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
578bfd0a 421 PkgFileName = File;
ad00ae81 422 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 423 Cache.HeaderP->PackageFileCount++;
b2e465d6 424
578bfd0a
AL
425 if (CurrentFile->FileName == 0)
426 return false;
404ec98e 427
ddc1d8d0 428 if (Progress != 0)
b2e465d6 429 Progress->SubProgress(Index.Size());
8efa2a3b 430 return true;
578bfd0a
AL
431}
432 /*}}}*/
f55a958f
AL
433// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
434// ---------------------------------------------------------------------
435/* This is used to create handles to strings. Given the same text it
436 always returns the same number */
437unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
438 unsigned int Size)
439{
f9eec0e7
AL
440 /* We use a very small transient hash table here, this speeds up generation
441 by a fair amount on slower machines */
442 pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
443 if (Bucket != 0 &&
444 stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
445 return Bucket->String;
446
f55a958f
AL
447 // Search for an insertion point
448 pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
449 int Res = 1;
349cd3b8 450 map_ptrloc *Last = &Cache.HeaderP->StringList;
f55a958f
AL
451 for (; I != Cache.StringItemP; Last = &I->NextItem,
452 I = Cache.StringItemP + I->NextItem)
453 {
9c14e3d6 454 Res = stringcmp(S,S+Size,Cache.StrP + I->String);
f55a958f
AL
455 if (Res >= 0)
456 break;
457 }
458
459 // Match
460 if (Res == 0)
f9eec0e7
AL
461 {
462 Bucket = I;
0149949b 463 return I->String;
f9eec0e7 464 }
f55a958f
AL
465
466 // Get a structure
467 unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
468 if (Item == 0)
0149949b
AL
469 return 0;
470
f55a958f
AL
471 // Fill in the structure
472 pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
473 ItemP->NextItem = I - Cache.StringItemP;
474 *Last = Item;
475 ItemP->String = Map.WriteString(S,Size);
476 if (ItemP->String == 0)
0149949b 477 return 0;
f55a958f 478
f9eec0e7 479 Bucket = ItemP;
0149949b 480 return ItemP->String;
f55a958f
AL
481}
482 /*}}}*/
b35d2f5f 483
b2e465d6 484// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 485// ---------------------------------------------------------------------
b2e465d6
AL
486/* This just verifies that each file in the list of index files exists,
487 has matching attributes with the cache and the cache does not have
488 any extra files. */
e7b470ee
AL
489static bool CheckValidity(string CacheFile, FileIterator Start,
490 FileIterator End,MMap **OutMap = 0)
b35d2f5f 491{
b2e465d6
AL
492 // No file, certainly invalid
493 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
b35d2f5f
AL
494 return false;
495
b2e465d6 496 // Map it
b35d2f5f 497 FileFd CacheF(CacheFile,FileFd::ReadOnly);
b2e465d6 498 SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
b35d2f5f 499 pkgCache Cache(Map);
b2e465d6 500 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f
AL
501 {
502 _error->Discard();
503 return false;
504 }
b35d2f5f 505
b2e465d6
AL
506 /* Now we check every index file, see if it is in the cache,
507 verify the IMS data and check that it is on the disk too.. */
508 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
509 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
510 for (; Start != End; Start++)
a77ad7c3 511 {
b2e465d6
AL
512 if ((*Start)->HasPackages() == false)
513 continue;
a77ad7c3 514
b2e465d6 515 if ((*Start)->Exists() == false)
b35d2f5f 516 {
b2e465d6
AL
517 _error->WarningE("stat",_("Couldn't stat source package list %s"),
518 (*Start)->Describe().c_str());
519 continue;
b35d2f5f 520 }
b2e465d6
AL
521
522 // FindInCache is also expected to do an IMS check.
523 pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
524 if (File.end() == true)
b35d2f5f 525 return false;
b2e465d6
AL
526
527 Visited[File->ID] = true;
b35d2f5f
AL
528 }
529
b2e465d6
AL
530 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
531 if (Visited[I] == false)
532 return false;
b35d2f5f 533
b35d2f5f
AL
534 if (_error->PendingError() == true)
535 {
536 _error->Discard();
537 return false;
538 }
b35d2f5f 539
b2e465d6
AL
540 if (OutMap != 0)
541 *OutMap = Map.UnGuard();
b35d2f5f
AL
542 return true;
543}
544 /*}}}*/
b2e465d6 545// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 546// ---------------------------------------------------------------------
b2e465d6
AL
547/* Size is kind of an abstract notion that is only used for the progress
548 meter */
e7b470ee 549static unsigned long ComputeSize(FileIterator Start,FileIterator End)
b35d2f5f 550{
b2e465d6
AL
551 unsigned long TotalSize = 0;
552 for (; Start != End; Start++)
b35d2f5f 553 {
b2e465d6
AL
554 if ((*Start)->HasPackages() == false)
555 continue;
556 TotalSize += (*Start)->Size();
b35d2f5f 557 }
b2e465d6 558 return TotalSize;
2d11135a
AL
559}
560 /*}}}*/
b2e465d6 561// BuildCache - Merge the list of index files into the cache /*{{{*/
2d11135a 562// ---------------------------------------------------------------------
b2e465d6
AL
563/* */
564static bool BuildCache(pkgCacheGenerator &Gen,
565 OpProgress &Progress,
566 unsigned long &CurrentSize,unsigned long TotalSize,
e7b470ee 567 FileIterator Start, FileIterator End)
2d11135a 568{
b2e465d6 569 for (; Start != End; Start++)
2d11135a 570 {
b2e465d6 571 if ((*Start)->HasPackages() == false)
2d11135a
AL
572 continue;
573
b2e465d6 574 if ((*Start)->Exists() == false)
2d11135a 575 continue;
b2e465d6 576
a77ad7c3
AL
577 if ((*Start)->FindInCache(Gen.GetCache()).end() == false)
578 {
579 _error->Warning("Duplicate sources.list entry %s",
580 (*Start)->Describe().c_str());
581 continue;
582 }
583
b2e465d6
AL
584 unsigned long Size = (*Start)->Size();
585 Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading Package Lists"));
586 CurrentSize += Size;
2d11135a 587
b2e465d6
AL
588 if ((*Start)->Merge(Gen,Progress) == false)
589 return false;
590 }
2d11135a 591
b35d2f5f
AL
592 return true;
593}
594 /*}}}*/
b2e465d6 595// MakeStatusCache - Construct the status cache /*{{{*/
b35d2f5f 596// ---------------------------------------------------------------------
b2e465d6
AL
597/* This makes sure that the status cache (the cache that has all
598 index files from the sources list and all local ones) is ready
599 to be mmaped. If OutMap is not zero then a MMap object representing
600 the cache will be stored there. This is pretty much mandetory if you
601 are using AllowMem. AllowMem lets the function be run as non-root
602 where it builds the cache 'fast' into a memory buffer. */
603bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
604 MMap **OutMap,bool AllowMem)
b35d2f5f 605{
a77ad7c3 606 unsigned long MapSize = _config->FindI("APT::Cache-Limit",6*1024*1024);
67db871e 607
b2e465d6
AL
608 vector<pkgIndexFile *> Files(List.begin(),List.end());
609 unsigned long EndOfSource = Files.size();
610 if (_system->AddStatusFiles(Files) == false)
611 return false;
8ce4327b 612
b2e465d6 613 // Decide if we can write to the files..
3b5421b4 614 string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
b2e465d6
AL
615 string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
616
617 // Decide if we can write to the cache
618 bool Writeable = false;
619 if (CacheFile.empty() == false)
620 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
621 else
622 if (SrcCacheFile.empty() == false)
623 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
624
625 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
626 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
627
628 Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
629
630 // Cache is OK, Fin.
631 if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
632 {
633 Progress.OverallProgress(1,1,1,_("Reading Package Lists"));
634 return true;
635 }
636
637 /* At this point we know we need to reconstruct the package cache,
638 begin. */
639 SPtr<FileFd> CacheF;
640 SPtr<DynamicMMap> Map;
641 if (Writeable == true && CacheFile.empty() == false)
642 {
643 unlink(CacheFile.c_str());
644 CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
7a3c2ab0 645 fchmod(CacheF->Fd(),0644);
b2e465d6 646 Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
b35d2f5f
AL
647 if (_error->PendingError() == true)
648 return false;
b35d2f5f 649 }
b2e465d6 650 else
8ce4327b 651 {
b2e465d6
AL
652 // Just build it in memory..
653 Map = new DynamicMMap(MMap::Public,MapSize);
8ce4327b 654 }
b35d2f5f 655
b2e465d6 656 // Lets try the source cache.
b35d2f5f 657 unsigned long CurrentSize = 0;
b2e465d6
AL
658 unsigned long TotalSize = 0;
659 if (CheckValidity(SrcCacheFile,Files.begin(),
660 Files.begin()+EndOfSource) == true)
2d11135a 661 {
b2e465d6
AL
662 // Preload the map with the source cache
663 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
664 if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
665 SCacheF.Size()) == false)
666 return false;
667
668 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
2d11135a 669
b2e465d6
AL
670 // Build the status cache
671 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 672 if (_error->PendingError() == true)
b2e465d6
AL
673 return false;
674 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
675 Files.begin()+EndOfSource,Files.end()) == false)
676 return false;
677 }
678 else
2d11135a 679 {
b2e465d6 680 TotalSize = ComputeSize(Files.begin(),Files.end());
2d11135a 681
b2e465d6
AL
682 // Build the source cache
683 pkgCacheGenerator Gen(Map.Get(),&Progress);
684 if (_error->PendingError() == true)
685 return false;
686 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
687 Files.begin(),Files.begin()+EndOfSource) == false)
688 return false;
2d11135a 689
b2e465d6
AL
690 // Write it back
691 if (Writeable == true && SrcCacheFile.empty() == false)
2d11135a 692 {
b2e465d6
AL
693 FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
694 if (_error->PendingError() == true)
695 return false;
7a3c2ab0
AL
696
697 fchmod(SCacheF.Fd(),0644);
698
b2e465d6
AL
699 // Write out the main data
700 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
701 return _error->Error(_("IO Error saving source cache"));
702 SCacheF.Sync();
703
704 // Write out the proper header
705 Gen.GetCache().HeaderP->Dirty = false;
706 if (SCacheF.Seek(0) == false ||
707 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
708 return _error->Error(_("IO Error saving source cache"));
b2e465d6 709 Gen.GetCache().HeaderP->Dirty = true;
7a3c2ab0 710 SCacheF.Sync();
2d11135a
AL
711 }
712
b2e465d6
AL
713 // Build the status cache
714 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
715 Files.begin()+EndOfSource,Files.end()) == false)
716 return false;
2d11135a
AL
717 }
718
b2e465d6
AL
719 if (_error->PendingError() == true)
720 return false;
721 if (OutMap != 0)
2d11135a 722 {
b2e465d6 723 if (CacheF != 0)
2d11135a 724 {
b2e465d6
AL
725 delete Map.UnGuard();
726 *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
2d11135a 727 }
b2e465d6
AL
728 else
729 {
730 *OutMap = Map.UnGuard();
731 }
2d11135a
AL
732 }
733
b2e465d6
AL
734 return true;
735}
736 /*}}}*/
737// MakeOnlyStatusCache - Build a cache with just the status files /*{{{*/
738// ---------------------------------------------------------------------
739/* */
740bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
741{
742 unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
743 vector<pkgIndexFile *> Files;
744 unsigned long EndOfSource = Files.size();
745 if (_system->AddStatusFiles(Files) == false)
746 return false;
747
748 SPtr<DynamicMMap> Map;
749 Map = new DynamicMMap(MMap::Public,MapSize);
750 unsigned long CurrentSize = 0;
751 unsigned long TotalSize = 0;
752
753 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
754
755 // Build the status cache
756 Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
757 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 758 if (_error->PendingError() == true)
b2e465d6
AL
759 return false;
760 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
761 Files.begin()+EndOfSource,Files.end()) == false)
762 return false;
2d11135a 763
b2e465d6
AL
764 if (_error->PendingError() == true)
765 return false;
766 *OutMap = Map.UnGuard();
2d11135a 767
b2e465d6 768 return true;
2d11135a
AL
769}
770 /*}}}*/