]> git.saurik.com Git - apt.git/blame_incremental - apt-pkg/pkgcachegen.cc
* merged the auto-remove branch
[apt.git] / apt-pkg / pkgcachegen.cc
... / ...
CommitLineData
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
3// $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
13#ifdef __GNUG__
14#pragma implementation "apt-pkg/pkgcachegen.h"
15#endif
16
17#define APT_COMPATIBILITY 986
18
19#include <apt-pkg/pkgcachegen.h>
20#include <apt-pkg/error.h>
21#include <apt-pkg/version.h>
22#include <apt-pkg/progress.h>
23#include <apt-pkg/sourcelist.h>
24#include <apt-pkg/configuration.h>
25#include <apt-pkg/strutl.h>
26#include <apt-pkg/sptr.h>
27#include <apt-pkg/pkgsystem.h>
28
29#include <apt-pkg/tagfile.h>
30
31#include <apti18n.h>
32
33#include <vector>
34
35#include <sys/stat.h>
36#include <unistd.h>
37#include <errno.h>
38#include <stdio.h>
39#include <system.h>
40 /*}}}*/
41typedef vector<pkgIndexFile *>::iterator FileIterator;
42
43// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
44// ---------------------------------------------------------------------
45/* We set the diry flag and make sure that is written to the disk */
46pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
47 Map(*pMap), Cache(pMap,false), Progress(Prog),
48 FoundFileDeps(0)
49{
50 CurrentFile = 0;
51 memset(UniqHash,0,sizeof(UniqHash));
52
53 if (_error->PendingError() == true)
54 return;
55
56 if (Map.Size() == 0)
57 {
58 // Setup the map interface..
59 Cache.HeaderP = (pkgCache::Header *)Map.Data();
60 Map.RawAllocate(sizeof(pkgCache::Header));
61 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
62
63 // Starting header
64 *Cache.HeaderP = pkgCache::Header();
65 Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
66 Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
67 Cache.ReMap();
68 }
69 else
70 {
71 // Map directly from the existing file
72 Cache.ReMap();
73 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
74 if (Cache.VS != _system->VS)
75 {
76 _error->Error(_("Cache has an incompatible versioning system"));
77 return;
78 }
79 }
80
81 Cache.HeaderP->Dirty = true;
82 Map.Sync(0,sizeof(pkgCache::Header));
83}
84 /*}}}*/
85// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
86// ---------------------------------------------------------------------
87/* We sync the data then unset the dirty flag in two steps so as to
88 advoid a problem during a crash */
89pkgCacheGenerator::~pkgCacheGenerator()
90{
91 if (_error->PendingError() == true)
92 return;
93 if (Map.Sync() == false)
94 return;
95
96 Cache.HeaderP->Dirty = false;
97 Map.Sync(0,sizeof(pkgCache::Header));
98}
99 /*}}}*/
100// CacheGenerator::MergeList - Merge the package list /*{{{*/
101// ---------------------------------------------------------------------
102/* This provides the generation of the entries in the cache. Each loop
103 goes through a single package record from the underlying parse engine. */
104bool pkgCacheGenerator::MergeList(ListParser &List,
105 pkgCache::VerIterator *OutVer)
106{
107 List.Owner = this;
108
109 unsigned int Counter = 0;
110 while (List.Step() == true)
111 {
112 // Get a pointer to the package structure
113 string PackageName = List.Package();
114 if (PackageName.empty() == true)
115 return false;
116
117 pkgCache::PkgIterator Pkg;
118 if (NewPackage(Pkg,PackageName) == false)
119 return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
120 Counter++;
121 if (Counter % 100 == 0 && Progress != 0)
122 Progress->Progress(List.Offset());
123
124 /* Get a pointer to the version structure. We know the list is sorted
125 so we use that fact in the search. Insertion of new versions is
126 done with correct sorting */
127 string Version = List.Version();
128 if (Version.empty() == true)
129 {
130 // we first process the package, then the descriptions
131 // (this has the bonus that we get MMap error when we run out
132 // of MMap space)
133 if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
134 return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
135 PackageName.c_str());
136
137 // Find the right version to write the description
138 MD5SumValue CurMd5 = List.Description_md5();
139 pkgCache::VerIterator Ver = Pkg.VersionList();
140 map_ptrloc *LastVer = &Pkg->VersionList;
141
142 for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
143 {
144 pkgCache::DescIterator Desc = Ver.DescriptionList();
145 map_ptrloc *LastDesc = &Ver->DescriptionList;
146
147 for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++)
148 {
149
150 if (MD5SumValue(Desc.md5()) == CurMd5)
151 {
152 // Add new description
153 *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
154 Desc->ParentPkg = Pkg.Index();
155
156 if (NewFileDesc(Desc,List) == false)
157 return _error->Error(_("Error occured while processing %s (NewFileDesc1)"),PackageName.c_str());
158 break;
159 }
160 }
161 }
162
163 continue;
164 }
165
166 pkgCache::VerIterator Ver = Pkg.VersionList();
167 map_ptrloc *LastVer = &Pkg->VersionList;
168 int Res = 1;
169 for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
170 {
171 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
172 if (Res >= 0)
173 break;
174 }
175
176 /* We already have a version for this item, record that we
177 saw it */
178 unsigned long Hash = List.VersionHash();
179 if (Res == 0 && Ver->Hash == Hash)
180 {
181 if (List.UsePackage(Pkg,Ver) == false)
182 return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
183 PackageName.c_str());
184
185 if (NewFileVer(Ver,List) == false)
186 return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
187 PackageName.c_str());
188
189 // Read only a single record and return
190 if (OutVer != 0)
191 {
192 *OutVer = Ver;
193 FoundFileDeps |= List.HasFileDeps();
194 return true;
195 }
196
197 continue;
198 }
199
200 // Skip to the end of the same version set.
201 if (Res == 0)
202 {
203 for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
204 {
205 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
206 if (Res != 0)
207 break;
208 }
209 }
210
211 // Add a new version
212 *LastVer = NewVersion(Ver,Version,*LastVer);
213 Ver->ParentPkg = Pkg.Index();
214 Ver->Hash = Hash;
215
216 if (List.NewVersion(Ver) == false)
217 return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
218 PackageName.c_str());
219
220 if (List.UsePackage(Pkg,Ver) == false)
221 return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
222 PackageName.c_str());
223
224 if (NewFileVer(Ver,List) == false)
225 return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
226 PackageName.c_str());
227
228 // Read only a single record and return
229 if (OutVer != 0)
230 {
231 *OutVer = Ver;
232 FoundFileDeps |= List.HasFileDeps();
233 return true;
234 }
235
236 /* Record the Description data. Description data always exist in
237 Packages and Translation-* files. */
238 pkgCache::DescIterator Desc = Ver.DescriptionList();
239 map_ptrloc *LastDesc = &Ver->DescriptionList;
240
241 // Skip to the end of description set
242 for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
243
244 // Add new description
245 *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
246 Desc->ParentPkg = Pkg.Index();
247
248 if (NewFileDesc(Desc,List) == false)
249 return _error->Error(_("Error occured while processing %s (NewFileDesc2)"),PackageName.c_str());
250 }
251
252 FoundFileDeps |= List.HasFileDeps();
253
254 if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
255 return _error->Error(_("Wow, you exceeded the number of package "
256 "names this APT is capable of."));
257 if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
258 return _error->Error(_("Wow, you exceeded the number of versions "
259 "this APT is capable of."));
260 if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
261 return _error->Error(_("Wow, you exceeded the number of descriptions "
262 "this APT is capable of."));
263 if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
264 return _error->Error(_("Wow, you exceeded the number of dependencies "
265 "this APT is capable of."));
266 return true;
267}
268 /*}}}*/
269// CacheGenerator::MergeFileProvides - Merge file provides /*{{{*/
270// ---------------------------------------------------------------------
271/* If we found any file depends while parsing the main list we need to
272 resolve them. Since it is undesired to load the entire list of files
273 into the cache as virtual packages we do a two stage effort. MergeList
274 identifies the file depends and this creates Provdies for them by
275 re-parsing all the indexs. */
276bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
277{
278 List.Owner = this;
279
280 unsigned int Counter = 0;
281 while (List.Step() == true)
282 {
283 string PackageName = List.Package();
284 if (PackageName.empty() == true)
285 return false;
286 string Version = List.Version();
287 if (Version.empty() == true)
288 continue;
289
290 pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
291 if (Pkg.end() == true)
292 return _error->Error(_("Error occurred while processing %s (FindPkg)"),
293 PackageName.c_str());
294 Counter++;
295 if (Counter % 100 == 0 && Progress != 0)
296 Progress->Progress(List.Offset());
297
298 unsigned long Hash = List.VersionHash();
299 pkgCache::VerIterator Ver = Pkg.VersionList();
300 for (; Ver.end() == false; Ver++)
301 {
302 if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
303 {
304 if (List.CollectFileProvides(Cache,Ver) == false)
305 return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
306 break;
307 }
308 }
309
310 if (Ver.end() == true)
311 _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
312 }
313
314 return true;
315}
316 /*}}}*/
317// CacheGenerator::NewPackage - Add a new package /*{{{*/
318// ---------------------------------------------------------------------
319/* This creates a new package structure and adds it to the hash table */
320bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
321{
322 Pkg = Cache.FindPkg(Name);
323 if (Pkg.end() == false)
324 return true;
325
326 // Get a structure
327 unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
328 if (Package == 0)
329 return false;
330
331 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
332
333 // Insert it into the hash table
334 unsigned long Hash = Cache.Hash(Name);
335 Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
336 Cache.HeaderP->HashTable[Hash] = Package;
337
338 // Set the name and the ID
339 Pkg->Name = Map.WriteString(Name);
340 if (Pkg->Name == 0)
341 return false;
342 Pkg->ID = Cache.HeaderP->PackageCount++;
343
344 return true;
345}
346 /*}}}*/
347// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
348// ---------------------------------------------------------------------
349/* */
350bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
351 ListParser &List)
352{
353 if (CurrentFile == 0)
354 return true;
355
356 // Get a structure
357 unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
358 if (VerFile == 0)
359 return 0;
360
361 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
362 VF->File = CurrentFile - Cache.PkgFileP;
363
364 // Link it to the end of the list
365 map_ptrloc *Last = &Ver->FileList;
366 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
367 Last = &V->NextFile;
368 VF->NextFile = *Last;
369 *Last = VF.Index();
370
371 VF->Offset = List.Offset();
372 VF->Size = List.Size();
373 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
374 Cache.HeaderP->MaxVerFileSize = VF->Size;
375 Cache.HeaderP->VerFileCount++;
376
377 return true;
378}
379 /*}}}*/
380// CacheGenerator::NewVersion - Create a new Version /*{{{*/
381// ---------------------------------------------------------------------
382/* This puts a version structure in the linked list */
383unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
384 const string &VerStr,
385 unsigned long Next)
386{
387 // Get a structure
388 unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
389 if (Version == 0)
390 return 0;
391
392 // Fill it in
393 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
394 Ver->NextVer = Next;
395 Ver->ID = Cache.HeaderP->VersionCount++;
396 Ver->VerStr = Map.WriteString(VerStr);
397 if (Ver->VerStr == 0)
398 return 0;
399
400 return Version;
401}
402 /*}}}*/
403// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
404// ---------------------------------------------------------------------
405/* */
406bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
407 ListParser &List)
408{
409 if (CurrentFile == 0)
410 return true;
411
412 // Get a structure
413 unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
414 if (DescFile == 0)
415 return 0;
416
417 pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
418 DF->File = CurrentFile - Cache.PkgFileP;
419
420 // Link it to the end of the list
421 map_ptrloc *Last = &Desc->FileList;
422 for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
423 Last = &D->NextFile;
424
425 DF->NextFile = *Last;
426 *Last = DF.Index();
427
428 DF->Offset = List.Offset();
429 DF->Size = List.Size();
430 if (Cache.HeaderP->MaxDescFileSize < DF->Size)
431 Cache.HeaderP->MaxDescFileSize = DF->Size;
432 Cache.HeaderP->DescFileCount++;
433
434 return true;
435}
436 /*}}}*/
437// CacheGenerator::NewDescription - Create a new Description /*{{{*/
438// ---------------------------------------------------------------------
439/* This puts a description structure in the linked list */
440map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
441 const string &Lang, const MD5SumValue &md5sum,
442 map_ptrloc Next)
443{
444 // Get a structure
445 map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
446 if (Description == 0)
447 return 0;
448
449 // Fill it in
450 Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
451 Desc->NextDesc = Next;
452 Desc->ID = Cache.HeaderP->DescriptionCount++;
453 Desc->language_code = Map.WriteString(Lang);
454 Desc->md5sum = Map.WriteString(md5sum.Value());
455
456 return Description;
457}
458 /*}}}*/
459// ListParser::NewDepends - Create a dependency element /*{{{*/
460// ---------------------------------------------------------------------
461/* This creates a dependency element in the tree. It is linked to the
462 version and to the package that it is pointing to. */
463bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
464 const string &PackageName,
465 const string &Version,
466 unsigned int Op,
467 unsigned int Type)
468{
469 pkgCache &Cache = Owner->Cache;
470
471 // Get a structure
472 unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
473 if (Dependency == 0)
474 return false;
475
476 // Fill it in
477 pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
478 Dep->ParentVer = Ver.Index();
479 Dep->Type = Type;
480 Dep->CompareOp = Op;
481 Dep->ID = Cache.HeaderP->DependsCount++;
482
483 // Locate the target package
484 pkgCache::PkgIterator Pkg;
485 if (Owner->NewPackage(Pkg,PackageName) == false)
486 return false;
487
488 // Probe the reverse dependency list for a version string that matches
489 if (Version.empty() == false)
490 {
491/* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
492 if (I->Version != 0 && I.TargetVer() == Version)
493 Dep->Version = I->Version;*/
494 if (Dep->Version == 0)
495 if ((Dep->Version = WriteString(Version)) == 0)
496 return false;
497 }
498
499 // Link it to the package
500 Dep->Package = Pkg.Index();
501 Dep->NextRevDepends = Pkg->RevDepends;
502 Pkg->RevDepends = Dep.Index();
503
504 /* Link it to the version (at the end of the list)
505 Caching the old end point speeds up generation substantially */
506 if (OldDepVer != Ver)
507 {
508 OldDepLast = &Ver->DependsList;
509 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
510 OldDepLast = &D->NextDepends;
511 OldDepVer = Ver;
512 }
513
514 // Is it a file dependency?
515 if (PackageName[0] == '/')
516 FoundFileDeps = true;
517
518 Dep->NextDepends = *OldDepLast;
519 *OldDepLast = Dep.Index();
520 OldDepLast = &Dep->NextDepends;
521
522 return true;
523}
524 /*}}}*/
525// ListParser::NewProvides - Create a Provides element /*{{{*/
526// ---------------------------------------------------------------------
527/* */
528bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
529 const string &PackageName,
530 const string &Version)
531{
532 pkgCache &Cache = Owner->Cache;
533
534 // We do not add self referencing provides
535 if (Ver.ParentPkg().Name() == PackageName)
536 return true;
537
538 // Get a structure
539 unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
540 if (Provides == 0)
541 return false;
542 Cache.HeaderP->ProvidesCount++;
543
544 // Fill it in
545 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
546 Prv->Version = Ver.Index();
547 Prv->NextPkgProv = Ver->ProvidesList;
548 Ver->ProvidesList = Prv.Index();
549 if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
550 return false;
551
552 // Locate the target package
553 pkgCache::PkgIterator Pkg;
554 if (Owner->NewPackage(Pkg,PackageName) == false)
555 return false;
556
557 // Link it to the package
558 Prv->ParentPkg = Pkg.Index();
559 Prv->NextProvides = Pkg->ProvidesList;
560 Pkg->ProvidesList = Prv.Index();
561
562 return true;
563}
564 /*}}}*/
565// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
566// ---------------------------------------------------------------------
567/* This is used to select which file is to be associated with all newly
568 added versions. The caller is responsible for setting the IMS fields. */
569bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
570 const pkgIndexFile &Index,
571 unsigned long Flags)
572{
573 // Get some space for the structure
574 CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
575 if (CurrentFile == Cache.PkgFileP)
576 return false;
577
578 // Fill it in
579 CurrentFile->FileName = Map.WriteString(File);
580 CurrentFile->Site = WriteUniqString(Site);
581 CurrentFile->NextFile = Cache.HeaderP->FileList;
582 CurrentFile->Flags = Flags;
583 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
584 CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
585 PkgFileName = File;
586 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
587 Cache.HeaderP->PackageFileCount++;
588
589 if (CurrentFile->FileName == 0)
590 return false;
591
592 if (Progress != 0)
593 Progress->SubProgress(Index.Size());
594 return true;
595}
596 /*}}}*/
597// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
598// ---------------------------------------------------------------------
599/* This is used to create handles to strings. Given the same text it
600 always returns the same number */
601unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
602 unsigned int Size)
603{
604 /* We use a very small transient hash table here, this speeds up generation
605 by a fair amount on slower machines */
606 pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
607 if (Bucket != 0 &&
608 stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
609 return Bucket->String;
610
611 // Search for an insertion point
612 pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
613 int Res = 1;
614 map_ptrloc *Last = &Cache.HeaderP->StringList;
615 for (; I != Cache.StringItemP; Last = &I->NextItem,
616 I = Cache.StringItemP + I->NextItem)
617 {
618 Res = stringcmp(S,S+Size,Cache.StrP + I->String);
619 if (Res >= 0)
620 break;
621 }
622
623 // Match
624 if (Res == 0)
625 {
626 Bucket = I;
627 return I->String;
628 }
629
630 // Get a structure
631 unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
632 if (Item == 0)
633 return 0;
634
635 // Fill in the structure
636 pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
637 ItemP->NextItem = I - Cache.StringItemP;
638 *Last = Item;
639 ItemP->String = Map.WriteString(S,Size);
640 if (ItemP->String == 0)
641 return 0;
642
643 Bucket = ItemP;
644 return ItemP->String;
645}
646 /*}}}*/
647
648// CheckValidity - Check that a cache is up-to-date /*{{{*/
649// ---------------------------------------------------------------------
650/* This just verifies that each file in the list of index files exists,
651 has matching attributes with the cache and the cache does not have
652 any extra files. */
653static bool CheckValidity(const string &CacheFile, FileIterator Start,
654 FileIterator End,MMap **OutMap = 0)
655{
656 // No file, certainly invalid
657 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
658 return false;
659
660 // Map it
661 FileFd CacheF(CacheFile,FileFd::ReadOnly);
662 SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
663 pkgCache Cache(Map);
664 if (_error->PendingError() == true || Map->Size() == 0)
665 {
666 _error->Discard();
667 return false;
668 }
669
670 /* Now we check every index file, see if it is in the cache,
671 verify the IMS data and check that it is on the disk too.. */
672 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
673 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
674 for (; Start != End; Start++)
675 {
676 if ((*Start)->HasPackages() == false)
677 continue;
678
679 if ((*Start)->Exists() == false)
680 {
681 _error->WarningE("stat",_("Couldn't stat source package list %s"),
682 (*Start)->Describe().c_str());
683 continue;
684 }
685
686 // FindInCache is also expected to do an IMS check.
687 pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
688 if (File.end() == true)
689 return false;
690
691 Visited[File->ID] = true;
692 }
693
694 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
695 if (Visited[I] == false)
696 return false;
697
698 if (_error->PendingError() == true)
699 {
700 _error->Discard();
701 return false;
702 }
703
704 if (OutMap != 0)
705 *OutMap = Map.UnGuard();
706 return true;
707}
708 /*}}}*/
709// ComputeSize - Compute the total size of a bunch of files /*{{{*/
710// ---------------------------------------------------------------------
711/* Size is kind of an abstract notion that is only used for the progress
712 meter */
713static unsigned long ComputeSize(FileIterator Start,FileIterator End)
714{
715 unsigned long TotalSize = 0;
716 for (; Start != End; Start++)
717 {
718 if ((*Start)->HasPackages() == false)
719 continue;
720 TotalSize += (*Start)->Size();
721 }
722 return TotalSize;
723}
724 /*}}}*/
725// BuildCache - Merge the list of index files into the cache /*{{{*/
726// ---------------------------------------------------------------------
727/* */
728static bool BuildCache(pkgCacheGenerator &Gen,
729 OpProgress &Progress,
730 unsigned long &CurrentSize,unsigned long TotalSize,
731 FileIterator Start, FileIterator End)
732{
733 FileIterator I;
734 for (I = Start; I != End; I++)
735 {
736 if ((*I)->HasPackages() == false)
737 continue;
738
739 if ((*I)->Exists() == false)
740 continue;
741
742 if ((*I)->FindInCache(Gen.GetCache()).end() == false)
743 {
744 _error->Warning("Duplicate sources.list entry %s",
745 (*I)->Describe().c_str());
746 continue;
747 }
748
749 unsigned long Size = (*I)->Size();
750 Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
751 CurrentSize += Size;
752
753 if ((*I)->Merge(Gen,Progress) == false)
754 return false;
755 }
756
757 if (Gen.HasFileDeps() == true)
758 {
759 Progress.Done();
760 TotalSize = ComputeSize(Start, End);
761 CurrentSize = 0;
762 for (I = Start; I != End; I++)
763 {
764 unsigned long Size = (*I)->Size();
765 Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
766 CurrentSize += Size;
767 if ((*I)->MergeFileProvides(Gen,Progress) == false)
768 return false;
769 }
770 }
771
772 return true;
773}
774 /*}}}*/
775// MakeStatusCache - Construct the status cache /*{{{*/
776// ---------------------------------------------------------------------
777/* This makes sure that the status cache (the cache that has all
778 index files from the sources list and all local ones) is ready
779 to be mmaped. If OutMap is not zero then a MMap object representing
780 the cache will be stored there. This is pretty much mandetory if you
781 are using AllowMem. AllowMem lets the function be run as non-root
782 where it builds the cache 'fast' into a memory buffer. */
783bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
784 MMap **OutMap,bool AllowMem)
785{
786 unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
787
788 vector<pkgIndexFile *> Files;
789 for (vector<metaIndex *>::const_iterator i = List.begin();
790 i != List.end();
791 i++)
792 {
793 vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
794 for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
795 j != Indexes->end();
796 j++)
797 Files.push_back (*j);
798 }
799
800 unsigned long EndOfSource = Files.size();
801 if (_system->AddStatusFiles(Files) == false)
802 return false;
803
804 // Decide if we can write to the files..
805 string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
806 string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
807
808 // Decide if we can write to the cache
809 bool Writeable = false;
810 if (CacheFile.empty() == false)
811 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
812 else
813 if (SrcCacheFile.empty() == false)
814 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
815
816 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
817 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
818
819 Progress.OverallProgress(0,1,1,_("Reading package lists"));
820
821 // Cache is OK, Fin.
822 if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
823 {
824 Progress.OverallProgress(1,1,1,_("Reading package lists"));
825 return true;
826 }
827
828 /* At this point we know we need to reconstruct the package cache,
829 begin. */
830 SPtr<FileFd> CacheF;
831 SPtr<DynamicMMap> Map;
832 if (Writeable == true && CacheFile.empty() == false)
833 {
834 unlink(CacheFile.c_str());
835 CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
836 fchmod(CacheF->Fd(),0644);
837 Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
838 if (_error->PendingError() == true)
839 return false;
840 }
841 else
842 {
843 // Just build it in memory..
844 Map = new DynamicMMap(MMap::Public,MapSize);
845 }
846
847 // Lets try the source cache.
848 unsigned long CurrentSize = 0;
849 unsigned long TotalSize = 0;
850 if (CheckValidity(SrcCacheFile,Files.begin(),
851 Files.begin()+EndOfSource) == true)
852 {
853 // Preload the map with the source cache
854 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
855 if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
856 SCacheF.Size()) == false)
857 return false;
858
859 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
860
861 // Build the status cache
862 pkgCacheGenerator Gen(Map.Get(),&Progress);
863 if (_error->PendingError() == true)
864 return false;
865 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
866 Files.begin()+EndOfSource,Files.end()) == false)
867 return false;
868 }
869 else
870 {
871 TotalSize = ComputeSize(Files.begin(),Files.end());
872
873 // Build the source cache
874 pkgCacheGenerator Gen(Map.Get(),&Progress);
875 if (_error->PendingError() == true)
876 return false;
877 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
878 Files.begin(),Files.begin()+EndOfSource) == false)
879 return false;
880
881 // Write it back
882 if (Writeable == true && SrcCacheFile.empty() == false)
883 {
884 FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
885 if (_error->PendingError() == true)
886 return false;
887
888 fchmod(SCacheF.Fd(),0644);
889
890 // Write out the main data
891 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
892 return _error->Error(_("IO Error saving source cache"));
893 SCacheF.Sync();
894
895 // Write out the proper header
896 Gen.GetCache().HeaderP->Dirty = false;
897 if (SCacheF.Seek(0) == false ||
898 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
899 return _error->Error(_("IO Error saving source cache"));
900 Gen.GetCache().HeaderP->Dirty = true;
901 SCacheF.Sync();
902 }
903
904 // Build the status cache
905 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
906 Files.begin()+EndOfSource,Files.end()) == false)
907 return false;
908 }
909
910 if (_error->PendingError() == true)
911 return false;
912 if (OutMap != 0)
913 {
914 if (CacheF != 0)
915 {
916 delete Map.UnGuard();
917 *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
918 }
919 else
920 {
921 *OutMap = Map.UnGuard();
922 }
923 }
924
925 return true;
926}
927 /*}}}*/
928// MakeOnlyStatusCache - Build a cache with just the status files /*{{{*/
929// ---------------------------------------------------------------------
930/* */
931bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
932{
933 unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
934 vector<pkgIndexFile *> Files;
935 unsigned long EndOfSource = Files.size();
936 if (_system->AddStatusFiles(Files) == false)
937 return false;
938
939 SPtr<DynamicMMap> Map;
940 Map = new DynamicMMap(MMap::Public,MapSize);
941 unsigned long CurrentSize = 0;
942 unsigned long TotalSize = 0;
943
944 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
945
946 // Build the status cache
947 Progress.OverallProgress(0,1,1,_("Reading package lists"));
948 pkgCacheGenerator Gen(Map.Get(),&Progress);
949 if (_error->PendingError() == true)
950 return false;
951 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
952 Files.begin()+EndOfSource,Files.end()) == false)
953 return false;
954
955 if (_error->PendingError() == true)
956 return false;
957 *OutMap = Map.UnGuard();
958
959 return true;
960}
961 /*}}}*/