]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
Work better if there are duplicate sources.list entries.
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
a77ad7c3 3// $Id: pkgcachegen.cc,v 1.47 2001/03/04 00:12:41 jgg Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
6c139d6e 13#ifdef __GNUG__
094a497d 14#pragma implementation "apt-pkg/pkgcachegen.h"
6c139d6e
AL
15#endif
16
b2e465d6
AL
17#define APT_COMPATIBILITY 986
18
094a497d
AL
19#include <apt-pkg/pkgcachegen.h>
20#include <apt-pkg/error.h>
21#include <apt-pkg/version.h>
b35d2f5f
AL
22#include <apt-pkg/progress.h>
23#include <apt-pkg/sourcelist.h>
24#include <apt-pkg/configuration.h>
cdcc6d34 25#include <apt-pkg/strutl.h>
b2e465d6
AL
26#include <apt-pkg/sptr.h>
27#include <apt-pkg/pkgsystem.h>
578bfd0a 28
b2e465d6
AL
29#include <apti18n.h>
30
578bfd0a
AL
31#include <sys/stat.h>
32#include <unistd.h>
803fafcb 33#include <errno.h>
7ef72446 34#include <stdio.h>
1ae93c94 35#include <system.h>
578bfd0a
AL
36 /*}}}*/
37
38// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
39// ---------------------------------------------------------------------
40/* We set the diry flag and make sure that is written to the disk */
b2e465d6
AL
41pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
42 Map(*pMap), Cache(pMap,false), Progress(Prog)
578bfd0a 43{
ddc1d8d0 44 CurrentFile = 0;
b2e465d6 45 memset(UniqHash,0,sizeof(UniqHash));
ddc1d8d0 46
578bfd0a
AL
47 if (_error->PendingError() == true)
48 return;
b2e465d6 49
578bfd0a
AL
50 if (Map.Size() == 0)
51 {
b2e465d6
AL
52 // Setup the map interface..
53 Cache.HeaderP = (pkgCache::Header *)Map.Data();
578bfd0a 54 Map.RawAllocate(sizeof(pkgCache::Header));
b2e465d6
AL
55 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
56
57 // Starting header
578bfd0a 58 *Cache.HeaderP = pkgCache::Header();
b2e465d6
AL
59 Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
60 Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
61 Cache.ReMap();
578bfd0a 62 }
b2e465d6
AL
63 else
64 {
65 // Map directly from the existing file
66 Cache.ReMap();
67 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
68 if (Cache.VS != _system->VS)
69 {
70 _error->Error(_("Cache has an incompatible versioning system"));
71 return;
72 }
73 }
74
578bfd0a
AL
75 Cache.HeaderP->Dirty = true;
76 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
77}
78 /*}}}*/
79// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
80// ---------------------------------------------------------------------
81/* We sync the data then unset the dirty flag in two steps so as to
82 advoid a problem during a crash */
83pkgCacheGenerator::~pkgCacheGenerator()
84{
85 if (_error->PendingError() == true)
86 return;
87 if (Map.Sync() == false)
88 return;
89
90 Cache.HeaderP->Dirty = false;
91 Map.Sync(0,sizeof(pkgCache::Header));
92}
93 /*}}}*/
94// CacheGenerator::MergeList - Merge the package list /*{{{*/
95// ---------------------------------------------------------------------
96/* This provides the generation of the entries in the cache. Each loop
97 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
98bool pkgCacheGenerator::MergeList(ListParser &List,
99 pkgCache::VerIterator *OutVer)
578bfd0a
AL
100{
101 List.Owner = this;
0149949b 102
f9eec0e7 103 unsigned int Counter = 0;
0149949b 104 while (List.Step() == true)
578bfd0a
AL
105 {
106 // Get a pointer to the package structure
9ddf7030 107 string PackageName = List.Package();
65a1e968
AL
108 if (PackageName.empty() == true)
109 return false;
110
9ddf7030 111 pkgCache::PkgIterator Pkg;
8efa2a3b 112 if (NewPackage(Pkg,PackageName) == false)
b2e465d6 113 return _error->Error(_("Error occured while processing %s (NewPackage)"),PackageName.c_str());
a246f2dc 114 Counter++;
ddc1d8d0
AL
115 if (Counter % 100 == 0 && Progress != 0)
116 Progress->Progress(List.Offset());
8ce4327b 117
578bfd0a
AL
118 /* Get a pointer to the version structure. We know the list is sorted
119 so we use that fact in the search. Insertion of new versions is
120 done with correct sorting */
121 string Version = List.Version();
f55a958f
AL
122 if (Version.empty() == true)
123 {
124 if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
b2e465d6 125 return _error->Error(_("Error occured while processing %s (UsePackage1)"),PackageName.c_str());
f55a958f
AL
126 continue;
127 }
128
578bfd0a 129 pkgCache::VerIterator Ver = Pkg.VersionList();
349cd3b8 130 map_ptrloc *Last = &Pkg->VersionList;
2246928b 131 int Res = 1;
f55a958f 132 for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
578bfd0a 133 {
b2e465d6 134 Res = Cache.VS->DoCmpVersion(Version.begin(),Version.end(),Ver.VerStr(),
578bfd0a
AL
135 Ver.VerStr() + strlen(Ver.VerStr()));
136 if (Res >= 0)
137 break;
138 }
139
140 /* We already have a version for this item, record that we
141 saw it */
204fbdcc
AL
142 unsigned long Hash = List.VersionHash();
143 if (Res == 0 && Ver->Hash == Hash)
578bfd0a 144 {
f55a958f 145 if (List.UsePackage(Pkg,Ver) == false)
b2e465d6 146 return _error->Error(_("Error occured while processing %s (UsePackage2)"),PackageName.c_str());
f78439bf 147
578bfd0a 148 if (NewFileVer(Ver,List) == false)
b2e465d6 149 return _error->Error(_("Error occured while processing %s (NewFileVer1)"),PackageName.c_str());
578bfd0a 150
ddc1d8d0
AL
151 // Read only a single record and return
152 if (OutVer != 0)
153 {
154 *OutVer = Ver;
155 return true;
156 }
157
578bfd0a
AL
158 continue;
159 }
160
204fbdcc
AL
161 // Skip to the end of the same version set.
162 if (Res == 0)
163 {
164 for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
165 {
b2e465d6 166 Res = Cache.VS->DoCmpVersion(Version.begin(),Version.end(),Ver.VerStr(),
204fbdcc
AL
167 Ver.VerStr() + strlen(Ver.VerStr()));
168 if (Res != 0)
169 break;
170 }
171 }
172
578bfd0a 173 // Add a new version
f55a958f
AL
174 *Last = NewVersion(Ver,Version,*Last);
175 Ver->ParentPkg = Pkg.Index();
204fbdcc 176 Ver->Hash = Hash;
578bfd0a 177 if (List.NewVersion(Ver) == false)
b2e465d6 178 return _error->Error(_("Error occured while processing %s (NewVersion1)"),PackageName.c_str());
0149949b 179
f55a958f 180 if (List.UsePackage(Pkg,Ver) == false)
b2e465d6 181 return _error->Error(_("Error occured while processing %s (UsePackage3)"),PackageName.c_str());
f55a958f 182
578bfd0a 183 if (NewFileVer(Ver,List) == false)
b2e465d6 184 return _error->Error(_("Error occured while processing %s (NewVersion2)"),PackageName.c_str());
ddc1d8d0
AL
185
186 // Read only a single record and return
187 if (OutVer != 0)
188 {
189 *OutVer = Ver;
190 return true;
191 }
578bfd0a 192 }
0149949b 193
578bfd0a
AL
194 return true;
195}
196 /*}}}*/
197// CacheGenerator::NewPackage - Add a new package /*{{{*/
198// ---------------------------------------------------------------------
199/* This creates a new package structure and adds it to the hash table */
f55a958f 200bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
578bfd0a 201{
8efa2a3b
AL
202 Pkg = Cache.FindPkg(Name);
203 if (Pkg.end() == false)
204 return true;
205
578bfd0a
AL
206 // Get a structure
207 unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
208 if (Package == 0)
209 return false;
210
f55a958f 211 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
578bfd0a
AL
212
213 // Insert it into the hash table
f55a958f 214 unsigned long Hash = Cache.Hash(Name);
578bfd0a
AL
215 Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
216 Cache.HeaderP->HashTable[Hash] = Package;
217
218 // Set the name and the ID
219 Pkg->Name = Map.WriteString(Name);
220 if (Pkg->Name == 0)
221 return false;
222 Pkg->ID = Cache.HeaderP->PackageCount++;
223
224 return true;
225}
226 /*}}}*/
227// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
228// ---------------------------------------------------------------------
229/* */
f55a958f 230bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
231 ListParser &List)
232{
ddc1d8d0
AL
233 if (CurrentFile == 0)
234 return true;
235
dcb79bae
AL
236 // Get a structure
237 unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
238 if (VerFile == 0)
239 return 0;
240
241 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
242 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
243
244 // Link it to the end of the list
349cd3b8 245 map_ptrloc *Last = &Ver->FileList;
03e39e59
AL
246 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
247 Last = &V->NextFile;
248 VF->NextFile = *Last;
249 *Last = VF.Index();
250
dcb79bae
AL
251 VF->Offset = List.Offset();
252 VF->Size = List.Size();
ad00ae81
AL
253 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
254 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
255 Cache.HeaderP->VerFileCount++;
256
f55a958f 257 return true;
578bfd0a
AL
258}
259 /*}}}*/
260// CacheGenerator::NewVersion - Create a new Version /*{{{*/
261// ---------------------------------------------------------------------
f55a958f 262/* This puts a version structure in the linked list */
578bfd0a 263unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
f55a958f 264 string VerStr,
578bfd0a
AL
265 unsigned long Next)
266{
f55a958f
AL
267 // Get a structure
268 unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
269 if (Version == 0)
0149949b 270 return 0;
f55a958f
AL
271
272 // Fill it in
273 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
f55a958f
AL
274 Ver->NextVer = Next;
275 Ver->ID = Cache.HeaderP->VersionCount++;
276 Ver->VerStr = Map.WriteString(VerStr);
277 if (Ver->VerStr == 0)
0149949b 278 return 0;
f55a958f 279
0149949b 280 return Version;
578bfd0a
AL
281}
282 /*}}}*/
dcb79bae
AL
283// ListParser::NewDepends - Create a dependency element /*{{{*/
284// ---------------------------------------------------------------------
285/* This creates a dependency element in the tree. It is linked to the
286 version and to the package that it is pointing to. */
287bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
9ddf7030
AL
288 string PackageName,
289 string Version,
dcb79bae
AL
290 unsigned int Op,
291 unsigned int Type)
292{
293 pkgCache &Cache = Owner->Cache;
294
295 // Get a structure
296 unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
297 if (Dependency == 0)
298 return false;
299
300 // Fill it in
301 pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
302 Dep->ParentVer = Ver.Index();
303 Dep->Type = Type;
304 Dep->CompareOp = Op;
305 Dep->ID = Cache.HeaderP->DependsCount++;
306
307 // Locate the target package
8efa2a3b
AL
308 pkgCache::PkgIterator Pkg;
309 if (Owner->NewPackage(Pkg,PackageName) == false)
310 return false;
dcb79bae
AL
311
312 // Probe the reverse dependency list for a version string that matches
313 if (Version.empty() == false)
314 {
b2e465d6 315/* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
dcb79bae 316 if (I->Version != 0 && I.TargetVer() == Version)
f9eec0e7 317 Dep->Version = I->Version;*/
dcb79bae
AL
318 if (Dep->Version == 0)
319 if ((Dep->Version = WriteString(Version)) == 0)
320 return false;
321 }
c1a22377 322
dcb79bae
AL
323 // Link it to the package
324 Dep->Package = Pkg.Index();
325 Dep->NextRevDepends = Pkg->RevDepends;
326 Pkg->RevDepends = Dep.Index();
327
c1a22377
AL
328 /* Link it to the version (at the end of the list)
329 Caching the old end point speeds up generation substantially */
f9eec0e7 330 if (OldDepVer != Ver)
c1a22377 331 {
f9eec0e7 332 OldDepLast = &Ver->DependsList;
c1a22377 333 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
f9eec0e7
AL
334 OldDepLast = &D->NextDepends;
335 OldDepVer = Ver;
c1a22377 336 }
dcb79bae 337
f9eec0e7
AL
338 Dep->NextDepends = *OldDepLast;
339 *OldDepLast = Dep.Index();
340 OldDepLast = &Dep->NextDepends;
c1a22377 341
dcb79bae
AL
342 return true;
343}
344 /*}}}*/
345// ListParser::NewProvides - Create a Provides element /*{{{*/
346// ---------------------------------------------------------------------
347/* */
348bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
8efa2a3b 349 string PackageName,
9ddf7030 350 string Version)
dcb79bae
AL
351{
352 pkgCache &Cache = Owner->Cache;
8efa2a3b
AL
353
354 // We do not add self referencing provides
355 if (Ver.ParentPkg().Name() == PackageName)
356 return true;
dcb79bae
AL
357
358 // Get a structure
359 unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
360 if (Provides == 0)
361 return false;
a7e66b17 362 Cache.HeaderP->ProvidesCount++;
dcb79bae
AL
363
364 // Fill it in
365 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
366 Prv->Version = Ver.Index();
367 Prv->NextPkgProv = Ver->ProvidesList;
368 Ver->ProvidesList = Prv.Index();
b2e465d6 369 if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
dcb79bae
AL
370 return false;
371
372 // Locate the target package
8efa2a3b
AL
373 pkgCache::PkgIterator Pkg;
374 if (Owner->NewPackage(Pkg,PackageName) == false)
375 return false;
dcb79bae
AL
376
377 // Link it to the package
378 Prv->ParentPkg = Pkg.Index();
379 Prv->NextProvides = Pkg->ProvidesList;
380 Pkg->ProvidesList = Prv.Index();
381
382 return true;
383}
384 /*}}}*/
578bfd0a
AL
385// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
386// ---------------------------------------------------------------------
387/* This is used to select which file is to be associated with all newly
b2e465d6
AL
388 added versions. The caller is responsible for setting the IMS fields. */
389bool pkgCacheGenerator::SelectFile(string File,string Site,
390 const pkgIndexFile &Index,
391 unsigned long Flags)
578bfd0a 392{
578bfd0a
AL
393 // Get some space for the structure
394 CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
395 if (CurrentFile == Cache.PkgFileP)
396 return false;
397
398 // Fill it in
399 CurrentFile->FileName = Map.WriteString(File);
b2e465d6 400 CurrentFile->Site = WriteUniqString(Site);
578bfd0a
AL
401 CurrentFile->NextFile = Cache.HeaderP->FileList;
402 CurrentFile->Flags = Flags;
e1b74f61 403 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
b2e465d6 404 CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
578bfd0a 405 PkgFileName = File;
ad00ae81 406 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 407 Cache.HeaderP->PackageFileCount++;
b2e465d6 408
578bfd0a
AL
409 if (CurrentFile->FileName == 0)
410 return false;
404ec98e 411
ddc1d8d0 412 if (Progress != 0)
b2e465d6 413 Progress->SubProgress(Index.Size());
8efa2a3b 414 return true;
578bfd0a
AL
415}
416 /*}}}*/
f55a958f
AL
417// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
418// ---------------------------------------------------------------------
419/* This is used to create handles to strings. Given the same text it
420 always returns the same number */
421unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
422 unsigned int Size)
423{
f9eec0e7
AL
424 /* We use a very small transient hash table here, this speeds up generation
425 by a fair amount on slower machines */
426 pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
427 if (Bucket != 0 &&
428 stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
429 return Bucket->String;
430
f55a958f
AL
431 // Search for an insertion point
432 pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
433 int Res = 1;
349cd3b8 434 map_ptrloc *Last = &Cache.HeaderP->StringList;
f55a958f
AL
435 for (; I != Cache.StringItemP; Last = &I->NextItem,
436 I = Cache.StringItemP + I->NextItem)
437 {
9c14e3d6 438 Res = stringcmp(S,S+Size,Cache.StrP + I->String);
f55a958f
AL
439 if (Res >= 0)
440 break;
441 }
442
443 // Match
444 if (Res == 0)
f9eec0e7
AL
445 {
446 Bucket = I;
0149949b 447 return I->String;
f9eec0e7 448 }
f55a958f
AL
449
450 // Get a structure
451 unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
452 if (Item == 0)
0149949b
AL
453 return 0;
454
f55a958f
AL
455 // Fill in the structure
456 pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
457 ItemP->NextItem = I - Cache.StringItemP;
458 *Last = Item;
459 ItemP->String = Map.WriteString(S,Size);
460 if (ItemP->String == 0)
0149949b 461 return 0;
f55a958f 462
f9eec0e7 463 Bucket = ItemP;
0149949b 464 return ItemP->String;
f55a958f
AL
465}
466 /*}}}*/
b35d2f5f 467
b2e465d6 468// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 469// ---------------------------------------------------------------------
b2e465d6
AL
470/* This just verifies that each file in the list of index files exists,
471 has matching attributes with the cache and the cache does not have
472 any extra files. */
473static bool CheckValidity(string CacheFile,pkgIndexFile **Start,
474 pkgIndexFile **End,MMap **OutMap = 0)
b35d2f5f 475{
b2e465d6
AL
476 // No file, certainly invalid
477 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
b35d2f5f
AL
478 return false;
479
b2e465d6 480 // Map it
b35d2f5f 481 FileFd CacheF(CacheFile,FileFd::ReadOnly);
b2e465d6 482 SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
b35d2f5f 483 pkgCache Cache(Map);
b2e465d6 484 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f
AL
485 {
486 _error->Discard();
487 return false;
488 }
b35d2f5f 489
b2e465d6
AL
490 /* Now we check every index file, see if it is in the cache,
491 verify the IMS data and check that it is on the disk too.. */
492 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
493 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
494 for (; Start != End; Start++)
a77ad7c3 495 {
b2e465d6
AL
496 if ((*Start)->HasPackages() == false)
497 continue;
a77ad7c3 498
b2e465d6 499 if ((*Start)->Exists() == false)
b35d2f5f 500 {
b2e465d6
AL
501 _error->WarningE("stat",_("Couldn't stat source package list %s"),
502 (*Start)->Describe().c_str());
503 continue;
b35d2f5f 504 }
b2e465d6
AL
505
506 // FindInCache is also expected to do an IMS check.
507 pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
508 if (File.end() == true)
b35d2f5f 509 return false;
b2e465d6
AL
510
511 Visited[File->ID] = true;
b35d2f5f
AL
512 }
513
b2e465d6
AL
514 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
515 if (Visited[I] == false)
516 return false;
b35d2f5f 517
b35d2f5f
AL
518 if (_error->PendingError() == true)
519 {
520 _error->Discard();
521 return false;
522 }
b35d2f5f 523
b2e465d6
AL
524 if (OutMap != 0)
525 *OutMap = Map.UnGuard();
b35d2f5f
AL
526 return true;
527}
528 /*}}}*/
b2e465d6 529// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 530// ---------------------------------------------------------------------
b2e465d6
AL
531/* Size is kind of an abstract notion that is only used for the progress
532 meter */
533static unsigned long ComputeSize(pkgIndexFile **Start,pkgIndexFile **End)
b35d2f5f 534{
b2e465d6
AL
535 unsigned long TotalSize = 0;
536 for (; Start != End; Start++)
b35d2f5f 537 {
b2e465d6
AL
538 if ((*Start)->HasPackages() == false)
539 continue;
540 TotalSize += (*Start)->Size();
b35d2f5f 541 }
b2e465d6 542 return TotalSize;
2d11135a
AL
543}
544 /*}}}*/
b2e465d6 545// BuildCache - Merge the list of index files into the cache /*{{{*/
2d11135a 546// ---------------------------------------------------------------------
b2e465d6
AL
547/* */
548static bool BuildCache(pkgCacheGenerator &Gen,
549 OpProgress &Progress,
550 unsigned long &CurrentSize,unsigned long TotalSize,
551 pkgIndexFile **Start,pkgIndexFile **End)
2d11135a 552{
b2e465d6 553 for (; Start != End; Start++)
2d11135a 554 {
b2e465d6 555 if ((*Start)->HasPackages() == false)
2d11135a
AL
556 continue;
557
b2e465d6 558 if ((*Start)->Exists() == false)
2d11135a 559 continue;
b2e465d6 560
a77ad7c3
AL
561 if ((*Start)->FindInCache(Gen.GetCache()).end() == false)
562 {
563 _error->Warning("Duplicate sources.list entry %s",
564 (*Start)->Describe().c_str());
565 continue;
566 }
567
b2e465d6
AL
568 unsigned long Size = (*Start)->Size();
569 Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading Package Lists"));
570 CurrentSize += Size;
2d11135a 571
b2e465d6
AL
572 if ((*Start)->Merge(Gen,Progress) == false)
573 return false;
574 }
2d11135a 575
b35d2f5f
AL
576 return true;
577}
578 /*}}}*/
b2e465d6 579// MakeStatusCache - Construct the status cache /*{{{*/
b35d2f5f 580// ---------------------------------------------------------------------
b2e465d6
AL
581/* This makes sure that the status cache (the cache that has all
582 index files from the sources list and all local ones) is ready
583 to be mmaped. If OutMap is not zero then a MMap object representing
584 the cache will be stored there. This is pretty much mandetory if you
585 are using AllowMem. AllowMem lets the function be run as non-root
586 where it builds the cache 'fast' into a memory buffer. */
587bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
588 MMap **OutMap,bool AllowMem)
b35d2f5f 589{
a77ad7c3 590 unsigned long MapSize = _config->FindI("APT::Cache-Limit",6*1024*1024);
67db871e 591
b2e465d6
AL
592 vector<pkgIndexFile *> Files(List.begin(),List.end());
593 unsigned long EndOfSource = Files.size();
594 if (_system->AddStatusFiles(Files) == false)
595 return false;
8ce4327b 596
b2e465d6 597 // Decide if we can write to the files..
3b5421b4 598 string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
b2e465d6
AL
599 string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
600
601 // Decide if we can write to the cache
602 bool Writeable = false;
603 if (CacheFile.empty() == false)
604 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
605 else
606 if (SrcCacheFile.empty() == false)
607 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
608
609 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
610 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
611
612 Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
613
614 // Cache is OK, Fin.
615 if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
616 {
617 Progress.OverallProgress(1,1,1,_("Reading Package Lists"));
618 return true;
619 }
620
621 /* At this point we know we need to reconstruct the package cache,
622 begin. */
623 SPtr<FileFd> CacheF;
624 SPtr<DynamicMMap> Map;
625 if (Writeable == true && CacheFile.empty() == false)
626 {
627 unlink(CacheFile.c_str());
628 CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
629 Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
b35d2f5f
AL
630 if (_error->PendingError() == true)
631 return false;
b35d2f5f 632 }
b2e465d6 633 else
8ce4327b 634 {
b2e465d6
AL
635 // Just build it in memory..
636 Map = new DynamicMMap(MMap::Public,MapSize);
8ce4327b 637 }
b35d2f5f 638
b2e465d6 639 // Lets try the source cache.
b35d2f5f 640 unsigned long CurrentSize = 0;
b2e465d6
AL
641 unsigned long TotalSize = 0;
642 if (CheckValidity(SrcCacheFile,Files.begin(),
643 Files.begin()+EndOfSource) == true)
2d11135a 644 {
b2e465d6
AL
645 // Preload the map with the source cache
646 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
647 if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
648 SCacheF.Size()) == false)
649 return false;
650
651 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
2d11135a 652
b2e465d6
AL
653 // Build the status cache
654 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 655 if (_error->PendingError() == true)
b2e465d6
AL
656 return false;
657 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
658 Files.begin()+EndOfSource,Files.end()) == false)
659 return false;
660 }
661 else
2d11135a 662 {
b2e465d6 663 TotalSize = ComputeSize(Files.begin(),Files.end());
2d11135a 664
b2e465d6
AL
665 // Build the source cache
666 pkgCacheGenerator Gen(Map.Get(),&Progress);
667 if (_error->PendingError() == true)
668 return false;
669 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
670 Files.begin(),Files.begin()+EndOfSource) == false)
671 return false;
2d11135a 672
b2e465d6
AL
673 // Write it back
674 if (Writeable == true && SrcCacheFile.empty() == false)
2d11135a 675 {
b2e465d6
AL
676 FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
677 if (_error->PendingError() == true)
678 return false;
679 // Write out the main data
680 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
681 return _error->Error(_("IO Error saving source cache"));
682 SCacheF.Sync();
683
684 // Write out the proper header
685 Gen.GetCache().HeaderP->Dirty = false;
686 if (SCacheF.Seek(0) == false ||
687 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
688 return _error->Error(_("IO Error saving source cache"));
689 SCacheF.Sync();
690 Gen.GetCache().HeaderP->Dirty = true;
2d11135a
AL
691 }
692
b2e465d6
AL
693 // Build the status cache
694 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
695 Files.begin()+EndOfSource,Files.end()) == false)
696 return false;
2d11135a
AL
697 }
698
b2e465d6
AL
699 if (_error->PendingError() == true)
700 return false;
701 if (OutMap != 0)
2d11135a 702 {
b2e465d6 703 if (CacheF != 0)
2d11135a 704 {
b2e465d6
AL
705 delete Map.UnGuard();
706 *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
2d11135a 707 }
b2e465d6
AL
708 else
709 {
710 *OutMap = Map.UnGuard();
711 }
2d11135a
AL
712 }
713
b2e465d6
AL
714 return true;
715}
716 /*}}}*/
717// MakeOnlyStatusCache - Build a cache with just the status files /*{{{*/
718// ---------------------------------------------------------------------
719/* */
720bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
721{
722 unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
723 vector<pkgIndexFile *> Files;
724 unsigned long EndOfSource = Files.size();
725 if (_system->AddStatusFiles(Files) == false)
726 return false;
727
728 SPtr<DynamicMMap> Map;
729 Map = new DynamicMMap(MMap::Public,MapSize);
730 unsigned long CurrentSize = 0;
731 unsigned long TotalSize = 0;
732
733 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
734
735 // Build the status cache
736 Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
737 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 738 if (_error->PendingError() == true)
b2e465d6
AL
739 return false;
740 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
741 Files.begin()+EndOfSource,Files.end()) == false)
742 return false;
2d11135a 743
b2e465d6
AL
744 if (_error->PendingError() == true)
745 return false;
746 *OutMap = Map.UnGuard();
2d11135a 747
b2e465d6 748 return true;
2d11135a
AL
749}
750 /*}}}*/