]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
- add --arch option for apt-ftparchive packages and contents commands
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
b2e465d6
AL
13#define APT_COMPATIBILITY 986
14
094a497d
AL
15#include <apt-pkg/pkgcachegen.h>
16#include <apt-pkg/error.h>
17#include <apt-pkg/version.h>
b35d2f5f
AL
18#include <apt-pkg/progress.h>
19#include <apt-pkg/sourcelist.h>
20#include <apt-pkg/configuration.h>
cdcc6d34 21#include <apt-pkg/strutl.h>
b2e465d6
AL
22#include <apt-pkg/sptr.h>
23#include <apt-pkg/pkgsystem.h>
5c0d3668 24#include <apt-pkg/macros.h>
578bfd0a 25
afb1e2e3
MV
26#include <apt-pkg/tagfile.h>
27
b2e465d6 28#include <apti18n.h>
e7b470ee
AL
29
30#include <vector>
31
578bfd0a
AL
32#include <sys/stat.h>
33#include <unistd.h>
803fafcb 34#include <errno.h>
7ef72446 35#include <stdio.h>
578bfd0a 36 /*}}}*/
e7b470ee 37typedef vector<pkgIndexFile *>::iterator FileIterator;
578bfd0a
AL
38
39// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
40// ---------------------------------------------------------------------
41/* We set the diry flag and make sure that is written to the disk */
b2e465d6 42pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45415543
AL
43 Map(*pMap), Cache(pMap,false), Progress(Prog),
44 FoundFileDeps(0)
578bfd0a 45{
ddc1d8d0 46 CurrentFile = 0;
b2e465d6 47 memset(UniqHash,0,sizeof(UniqHash));
ddc1d8d0 48
578bfd0a
AL
49 if (_error->PendingError() == true)
50 return;
b2e465d6 51
578bfd0a
AL
52 if (Map.Size() == 0)
53 {
b2e465d6
AL
54 // Setup the map interface..
55 Cache.HeaderP = (pkgCache::Header *)Map.Data();
c5f44afc
DK
56 if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
57 return;
58
b2e465d6 59 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
c5f44afc 60
b2e465d6 61 // Starting header
578bfd0a 62 *Cache.HeaderP = pkgCache::Header();
b2e465d6
AL
63 Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
64 Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
c5f44afc 65 Cache.ReMap();
578bfd0a 66 }
b2e465d6
AL
67 else
68 {
69 // Map directly from the existing file
70 Cache.ReMap();
71 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
72 if (Cache.VS != _system->VS)
73 {
74 _error->Error(_("Cache has an incompatible versioning system"));
75 return;
76 }
77 }
78
578bfd0a
AL
79 Cache.HeaderP->Dirty = true;
80 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
81}
82 /*}}}*/
83// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
84// ---------------------------------------------------------------------
85/* We sync the data then unset the dirty flag in two steps so as to
86 advoid a problem during a crash */
87pkgCacheGenerator::~pkgCacheGenerator()
88{
89 if (_error->PendingError() == true)
90 return;
91 if (Map.Sync() == false)
92 return;
93
94 Cache.HeaderP->Dirty = false;
95 Map.Sync(0,sizeof(pkgCache::Header));
96}
97 /*}}}*/
98// CacheGenerator::MergeList - Merge the package list /*{{{*/
99// ---------------------------------------------------------------------
100/* This provides the generation of the entries in the cache. Each loop
101 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
102bool pkgCacheGenerator::MergeList(ListParser &List,
103 pkgCache::VerIterator *OutVer)
578bfd0a
AL
104{
105 List.Owner = this;
0149949b 106
f9eec0e7 107 unsigned int Counter = 0;
0149949b 108 while (List.Step() == true)
578bfd0a
AL
109 {
110 // Get a pointer to the package structure
9ddf7030 111 string PackageName = List.Package();
65a1e968
AL
112 if (PackageName.empty() == true)
113 return false;
114
9ddf7030 115 pkgCache::PkgIterator Pkg;
8efa2a3b 116 if (NewPackage(Pkg,PackageName) == false)
6804503b 117 return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
a246f2dc 118 Counter++;
ddc1d8d0
AL
119 if (Counter % 100 == 0 && Progress != 0)
120 Progress->Progress(List.Offset());
8ce4327b 121
578bfd0a
AL
122 /* Get a pointer to the version structure. We know the list is sorted
123 so we use that fact in the search. Insertion of new versions is
124 done with correct sorting */
125 string Version = List.Version();
f55a958f
AL
126 if (Version.empty() == true)
127 {
770c32ec
MV
128 // we first process the package, then the descriptions
129 // (this has the bonus that we get MMap error when we run out
130 // of MMap space)
f55a958f 131 if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
6804503b 132 return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
7a3c2ab0 133 PackageName.c_str());
770c32ec 134
a52f938b
OS
135 // Find the right version to write the description
136 MD5SumValue CurMd5 = List.Description_md5();
137 pkgCache::VerIterator Ver = Pkg.VersionList();
138 map_ptrloc *LastVer = &Pkg->VersionList;
770c32ec 139
9ee47c29 140 for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
a52f938b
OS
141 {
142 pkgCache::DescIterator Desc = Ver.DescriptionList();
143 map_ptrloc *LastDesc = &Ver->DescriptionList;
c76c44b1 144 bool duplicate=false;
145
146 // don't add a new description if we have one for the given
147 // md5 && language
9ee47c29 148 for ( ; Desc.end() == false; Desc++)
c76c44b1 149 if (MD5SumValue(Desc.md5()) == CurMd5 &&
150 Desc.LanguageCode() == List.DescriptionLanguage())
151 duplicate=true;
152 if(duplicate)
153 continue;
154
155 for (Desc = Ver.DescriptionList();
9ee47c29 156 Desc.end() == false;
c76c44b1 157 LastDesc = &Desc->NextDesc, Desc++)
770c32ec 158 {
770c32ec
MV
159 if (MD5SumValue(Desc.md5()) == CurMd5)
160 {
a52f938b
OS
161 // Add new description
162 *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
163 Desc->ParentPkg = Pkg.Index();
770c32ec 164
c5f44afc 165 if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
f0f66a3d 166 return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
a52f938b
OS
167 break;
168 }
770c32ec 169 }
a52f938b 170 }
770c32ec 171
f55a958f
AL
172 continue;
173 }
174
578bfd0a 175 pkgCache::VerIterator Ver = Pkg.VersionList();
a52f938b 176 map_ptrloc *LastVer = &Pkg->VersionList;
2246928b 177 int Res = 1;
9ee47c29 178 for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
578bfd0a 179 {
c24972cb 180 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
578bfd0a
AL
181 if (Res >= 0)
182 break;
183 }
184
185 /* We already have a version for this item, record that we
186 saw it */
204fbdcc
AL
187 unsigned long Hash = List.VersionHash();
188 if (Res == 0 && Ver->Hash == Hash)
578bfd0a 189 {
f55a958f 190 if (List.UsePackage(Pkg,Ver) == false)
6804503b 191 return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
7a3c2ab0 192 PackageName.c_str());
f78439bf 193
578bfd0a 194 if (NewFileVer(Ver,List) == false)
6804503b 195 return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
7a3c2ab0 196 PackageName.c_str());
578bfd0a 197
ddc1d8d0
AL
198 // Read only a single record and return
199 if (OutVer != 0)
200 {
201 *OutVer = Ver;
45415543 202 FoundFileDeps |= List.HasFileDeps();
ddc1d8d0
AL
203 return true;
204 }
205
578bfd0a
AL
206 continue;
207 }
208
204fbdcc
AL
209 // Skip to the end of the same version set.
210 if (Res == 0)
211 {
9ee47c29 212 for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
204fbdcc 213 {
c24972cb 214 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
204fbdcc
AL
215 if (Res != 0)
216 break;
217 }
218 }
219
578bfd0a 220 // Add a new version
a52f938b 221 *LastVer = NewVersion(Ver,Version,*LastVer);
f55a958f 222 Ver->ParentPkg = Pkg.Index();
204fbdcc 223 Ver->Hash = Hash;
a52f938b 224
c5f44afc 225 if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false)
6804503b 226 return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
7a3c2ab0 227 PackageName.c_str());
0149949b 228
f55a958f 229 if (List.UsePackage(Pkg,Ver) == false)
6804503b 230 return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
7a3c2ab0 231 PackageName.c_str());
f55a958f 232
578bfd0a 233 if (NewFileVer(Ver,List) == false)
6804503b 234 return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
7a3c2ab0 235 PackageName.c_str());
ddc1d8d0
AL
236
237 // Read only a single record and return
238 if (OutVer != 0)
239 {
240 *OutVer = Ver;
45415543 241 FoundFileDeps |= List.HasFileDeps();
ddc1d8d0
AL
242 return true;
243 }
a52f938b
OS
244
245 /* Record the Description data. Description data always exist in
246 Packages and Translation-* files. */
247 pkgCache::DescIterator Desc = Ver.DescriptionList();
248 map_ptrloc *LastDesc = &Ver->DescriptionList;
249
250 // Skip to the end of description set
9ee47c29 251 for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
a52f938b
OS
252
253 // Add new description
254 *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
255 Desc->ParentPkg = Pkg.Index();
256
c5f44afc 257 if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
f0f66a3d 258 return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
578bfd0a 259 }
0149949b 260
45415543
AL
261 FoundFileDeps |= List.HasFileDeps();
262
6a3da7a6
AL
263 if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
264 return _error->Error(_("Wow, you exceeded the number of package "
265 "names this APT is capable of."));
266 if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
267 return _error->Error(_("Wow, you exceeded the number of versions "
268 "this APT is capable of."));
a52f938b
OS
269 if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
270 return _error->Error(_("Wow, you exceeded the number of descriptions "
271 "this APT is capable of."));
6a3da7a6
AL
272 if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
273 return _error->Error(_("Wow, you exceeded the number of dependencies "
274 "this APT is capable of."));
578bfd0a
AL
275 return true;
276}
277 /*}}}*/
45415543
AL
278// CacheGenerator::MergeFileProvides - Merge file provides /*{{{*/
279// ---------------------------------------------------------------------
280/* If we found any file depends while parsing the main list we need to
281 resolve them. Since it is undesired to load the entire list of files
282 into the cache as virtual packages we do a two stage effort. MergeList
283 identifies the file depends and this creates Provdies for them by
284 re-parsing all the indexs. */
285bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
286{
287 List.Owner = this;
288
289 unsigned int Counter = 0;
290 while (List.Step() == true)
291 {
292 string PackageName = List.Package();
293 if (PackageName.empty() == true)
294 return false;
295 string Version = List.Version();
296 if (Version.empty() == true)
297 continue;
298
299 pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
300 if (Pkg.end() == true)
6804503b 301 return _error->Error(_("Error occurred while processing %s (FindPkg)"),
45415543
AL
302 PackageName.c_str());
303 Counter++;
304 if (Counter % 100 == 0 && Progress != 0)
305 Progress->Progress(List.Offset());
306
307 unsigned long Hash = List.VersionHash();
308 pkgCache::VerIterator Ver = Pkg.VersionList();
9ee47c29 309 for (; Ver.end() == false; Ver++)
45415543
AL
310 {
311 if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
312 {
313 if (List.CollectFileProvides(Cache,Ver) == false)
6804503b 314 return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
45415543
AL
315 break;
316 }
317 }
318
319 if (Ver.end() == true)
320 _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
321 }
322
323 return true;
324}
325 /*}}}*/
578bfd0a
AL
326// CacheGenerator::NewPackage - Add a new package /*{{{*/
327// ---------------------------------------------------------------------
328/* This creates a new package structure and adds it to the hash table */
171c75f1 329bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
578bfd0a 330{
8efa2a3b
AL
331 Pkg = Cache.FindPkg(Name);
332 if (Pkg.end() == false)
333 return true;
a52f938b 334
578bfd0a
AL
335 // Get a structure
336 unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
337 if (Package == 0)
338 return false;
339
f55a958f 340 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
578bfd0a
AL
341
342 // Insert it into the hash table
f55a958f 343 unsigned long Hash = Cache.Hash(Name);
578bfd0a
AL
344 Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
345 Cache.HeaderP->HashTable[Hash] = Package;
346
347 // Set the name and the ID
348 Pkg->Name = Map.WriteString(Name);
349 if (Pkg->Name == 0)
350 return false;
351 Pkg->ID = Cache.HeaderP->PackageCount++;
352
353 return true;
354}
355 /*}}}*/
356// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
357// ---------------------------------------------------------------------
358/* */
f55a958f 359bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
360 ListParser &List)
361{
ddc1d8d0
AL
362 if (CurrentFile == 0)
363 return true;
364
dcb79bae
AL
365 // Get a structure
366 unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
367 if (VerFile == 0)
368 return 0;
369
370 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
371 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
372
373 // Link it to the end of the list
349cd3b8 374 map_ptrloc *Last = &Ver->FileList;
eb162ff7 375 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
03e39e59
AL
376 Last = &V->NextFile;
377 VF->NextFile = *Last;
378 *Last = VF.Index();
379
dcb79bae
AL
380 VF->Offset = List.Offset();
381 VF->Size = List.Size();
ad00ae81
AL
382 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
383 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
384 Cache.HeaderP->VerFileCount++;
385
f55a958f 386 return true;
578bfd0a
AL
387}
388 /*}}}*/
389// CacheGenerator::NewVersion - Create a new Version /*{{{*/
390// ---------------------------------------------------------------------
f55a958f 391/* This puts a version structure in the linked list */
578bfd0a 392unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
171c75f1 393 const string &VerStr,
578bfd0a
AL
394 unsigned long Next)
395{
f55a958f
AL
396 // Get a structure
397 unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
398 if (Version == 0)
0149949b 399 return 0;
f55a958f
AL
400
401 // Fill it in
402 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
f55a958f
AL
403 Ver->NextVer = Next;
404 Ver->ID = Cache.HeaderP->VersionCount++;
405 Ver->VerStr = Map.WriteString(VerStr);
406 if (Ver->VerStr == 0)
0149949b 407 return 0;
f55a958f 408
0149949b 409 return Version;
578bfd0a
AL
410}
411 /*}}}*/
a52f938b
OS
412// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
413// ---------------------------------------------------------------------
414/* */
415bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
416 ListParser &List)
417{
418 if (CurrentFile == 0)
419 return true;
420
421 // Get a structure
422 unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
423 if (DescFile == 0)
c5f44afc 424 return false;
770c32ec 425
a52f938b
OS
426 pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
427 DF->File = CurrentFile - Cache.PkgFileP;
770c32ec 428
a52f938b
OS
429 // Link it to the end of the list
430 map_ptrloc *Last = &Desc->FileList;
eb162ff7 431 for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
a52f938b 432 Last = &D->NextFile;
770c32ec 433
a52f938b
OS
434 DF->NextFile = *Last;
435 *Last = DF.Index();
436
437 DF->Offset = List.Offset();
438 DF->Size = List.Size();
439 if (Cache.HeaderP->MaxDescFileSize < DF->Size)
440 Cache.HeaderP->MaxDescFileSize = DF->Size;
441 Cache.HeaderP->DescFileCount++;
442
443 return true;
444}
445 /*}}}*/
446// CacheGenerator::NewDescription - Create a new Description /*{{{*/
447// ---------------------------------------------------------------------
448/* This puts a description structure in the linked list */
449map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
c76c44b1 450 const string &Lang,
451 const MD5SumValue &md5sum,
a52f938b
OS
452 map_ptrloc Next)
453{
454 // Get a structure
455 map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
456 if (Description == 0)
457 return 0;
458
459 // Fill it in
460 Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
461 Desc->NextDesc = Next;
462 Desc->ID = Cache.HeaderP->DescriptionCount++;
463 Desc->language_code = Map.WriteString(Lang);
464 Desc->md5sum = Map.WriteString(md5sum.Value());
c5f44afc
DK
465 if (Desc->language_code == 0 || Desc->md5sum == 0)
466 return 0;
a52f938b
OS
467
468 return Description;
469}
470 /*}}}*/
dcb79bae
AL
471// ListParser::NewDepends - Create a dependency element /*{{{*/
472// ---------------------------------------------------------------------
473/* This creates a dependency element in the tree. It is linked to the
474 version and to the package that it is pointing to. */
475bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
171c75f1
MV
476 const string &PackageName,
477 const string &Version,
dcb79bae
AL
478 unsigned int Op,
479 unsigned int Type)
480{
481 pkgCache &Cache = Owner->Cache;
482
483 // Get a structure
484 unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
485 if (Dependency == 0)
486 return false;
487
488 // Fill it in
489 pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
490 Dep->ParentVer = Ver.Index();
491 Dep->Type = Type;
492 Dep->CompareOp = Op;
493 Dep->ID = Cache.HeaderP->DependsCount++;
494
495 // Locate the target package
8efa2a3b
AL
496 pkgCache::PkgIterator Pkg;
497 if (Owner->NewPackage(Pkg,PackageName) == false)
498 return false;
dcb79bae
AL
499
500 // Probe the reverse dependency list for a version string that matches
501 if (Version.empty() == false)
502 {
b2e465d6 503/* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
dcb79bae 504 if (I->Version != 0 && I.TargetVer() == Version)
f9eec0e7 505 Dep->Version = I->Version;*/
dcb79bae
AL
506 if (Dep->Version == 0)
507 if ((Dep->Version = WriteString(Version)) == 0)
508 return false;
509 }
c1a22377 510
dcb79bae
AL
511 // Link it to the package
512 Dep->Package = Pkg.Index();
513 Dep->NextRevDepends = Pkg->RevDepends;
514 Pkg->RevDepends = Dep.Index();
515
c1a22377
AL
516 /* Link it to the version (at the end of the list)
517 Caching the old end point speeds up generation substantially */
f9eec0e7 518 if (OldDepVer != Ver)
c1a22377 519 {
f9eec0e7 520 OldDepLast = &Ver->DependsList;
eb162ff7 521 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
f9eec0e7
AL
522 OldDepLast = &D->NextDepends;
523 OldDepVer = Ver;
c1a22377 524 }
45415543
AL
525
526 // Is it a file dependency?
527 if (PackageName[0] == '/')
528 FoundFileDeps = true;
dcb79bae 529
f9eec0e7
AL
530 Dep->NextDepends = *OldDepLast;
531 *OldDepLast = Dep.Index();
532 OldDepLast = &Dep->NextDepends;
c1a22377 533
dcb79bae
AL
534 return true;
535}
536 /*}}}*/
537// ListParser::NewProvides - Create a Provides element /*{{{*/
538// ---------------------------------------------------------------------
539/* */
540bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
171c75f1
MV
541 const string &PackageName,
542 const string &Version)
dcb79bae
AL
543{
544 pkgCache &Cache = Owner->Cache;
8efa2a3b
AL
545
546 // We do not add self referencing provides
547 if (Ver.ParentPkg().Name() == PackageName)
548 return true;
dcb79bae
AL
549
550 // Get a structure
551 unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
552 if (Provides == 0)
553 return false;
a7e66b17 554 Cache.HeaderP->ProvidesCount++;
dcb79bae
AL
555
556 // Fill it in
557 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
558 Prv->Version = Ver.Index();
559 Prv->NextPkgProv = Ver->ProvidesList;
560 Ver->ProvidesList = Prv.Index();
b2e465d6 561 if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
dcb79bae
AL
562 return false;
563
564 // Locate the target package
8efa2a3b
AL
565 pkgCache::PkgIterator Pkg;
566 if (Owner->NewPackage(Pkg,PackageName) == false)
567 return false;
dcb79bae
AL
568
569 // Link it to the package
570 Prv->ParentPkg = Pkg.Index();
571 Prv->NextProvides = Pkg->ProvidesList;
572 Pkg->ProvidesList = Prv.Index();
573
574 return true;
575}
576 /*}}}*/
578bfd0a
AL
577// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
578// ---------------------------------------------------------------------
579/* This is used to select which file is to be associated with all newly
b2e465d6 580 added versions. The caller is responsible for setting the IMS fields. */
171c75f1 581bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
b2e465d6
AL
582 const pkgIndexFile &Index,
583 unsigned long Flags)
578bfd0a 584{
578bfd0a
AL
585 // Get some space for the structure
586 CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
587 if (CurrentFile == Cache.PkgFileP)
588 return false;
589
590 // Fill it in
591 CurrentFile->FileName = Map.WriteString(File);
b2e465d6 592 CurrentFile->Site = WriteUniqString(Site);
578bfd0a
AL
593 CurrentFile->NextFile = Cache.HeaderP->FileList;
594 CurrentFile->Flags = Flags;
e1b74f61 595 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
b2e465d6 596 CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
578bfd0a 597 PkgFileName = File;
ad00ae81 598 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 599 Cache.HeaderP->PackageFileCount++;
b2e465d6 600
578bfd0a
AL
601 if (CurrentFile->FileName == 0)
602 return false;
404ec98e 603
ddc1d8d0 604 if (Progress != 0)
b2e465d6 605 Progress->SubProgress(Index.Size());
8efa2a3b 606 return true;
578bfd0a
AL
607}
608 /*}}}*/
f55a958f
AL
609// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
610// ---------------------------------------------------------------------
611/* This is used to create handles to strings. Given the same text it
612 always returns the same number */
613unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
614 unsigned int Size)
615{
f9eec0e7
AL
616 /* We use a very small transient hash table here, this speeds up generation
617 by a fair amount on slower machines */
618 pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
619 if (Bucket != 0 &&
620 stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
621 return Bucket->String;
622
f55a958f
AL
623 // Search for an insertion point
624 pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
625 int Res = 1;
349cd3b8 626 map_ptrloc *Last = &Cache.HeaderP->StringList;
f55a958f
AL
627 for (; I != Cache.StringItemP; Last = &I->NextItem,
628 I = Cache.StringItemP + I->NextItem)
629 {
9c14e3d6 630 Res = stringcmp(S,S+Size,Cache.StrP + I->String);
f55a958f
AL
631 if (Res >= 0)
632 break;
633 }
634
635 // Match
636 if (Res == 0)
f9eec0e7
AL
637 {
638 Bucket = I;
0149949b 639 return I->String;
f9eec0e7 640 }
f55a958f
AL
641
642 // Get a structure
643 unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
644 if (Item == 0)
0149949b
AL
645 return 0;
646
f55a958f
AL
647 // Fill in the structure
648 pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
649 ItemP->NextItem = I - Cache.StringItemP;
650 *Last = Item;
651 ItemP->String = Map.WriteString(S,Size);
652 if (ItemP->String == 0)
0149949b 653 return 0;
f55a958f 654
f9eec0e7 655 Bucket = ItemP;
0149949b 656 return ItemP->String;
f55a958f
AL
657}
658 /*}}}*/
b2e465d6 659// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 660// ---------------------------------------------------------------------
b2e465d6
AL
661/* This just verifies that each file in the list of index files exists,
662 has matching attributes with the cache and the cache does not have
663 any extra files. */
171c75f1 664static bool CheckValidity(const string &CacheFile, FileIterator Start,
e7b470ee 665 FileIterator End,MMap **OutMap = 0)
b35d2f5f 666{
c8e572e3 667 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
b2e465d6
AL
668 // No file, certainly invalid
669 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
c8e572e3
MV
670 {
671 if (Debug == true)
672 std::clog << "CacheFile doesn't exist" << std::endl;
b35d2f5f 673 return false;
c8e572e3
MV
674 }
675
b2e465d6 676 // Map it
b35d2f5f 677 FileFd CacheF(CacheFile,FileFd::ReadOnly);
eb162ff7 678 SPtr<MMap> Map = new MMap(CacheF,0);
b35d2f5f 679 pkgCache Cache(Map);
b2e465d6 680 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f 681 {
c8e572e3
MV
682 if (Debug == true)
683 std::clog << "Errors are pending or Map is empty()" << std::endl;
b35d2f5f
AL
684 _error->Discard();
685 return false;
686 }
b35d2f5f 687
b2e465d6
AL
688 /* Now we check every index file, see if it is in the cache,
689 verify the IMS data and check that it is on the disk too.. */
690 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
691 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
692 for (; Start != End; Start++)
c8e572e3
MV
693 {
694 if (Debug == true)
695 std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
b2e465d6 696 if ((*Start)->HasPackages() == false)
c8e572e3
MV
697 {
698 if (Debug == true)
699 std::clog << "Has NO packages" << std::endl;
b2e465d6 700 continue;
c8e572e3 701 }
a77ad7c3 702
b2e465d6 703 if ((*Start)->Exists() == false)
b35d2f5f 704 {
a791a450 705#if 0 // mvo: we no longer give a message here (Default Sources spec)
b2e465d6
AL
706 _error->WarningE("stat",_("Couldn't stat source package list %s"),
707 (*Start)->Describe().c_str());
a791a450 708#endif
c8e572e3
MV
709 if (Debug == true)
710 std::clog << "file doesn't exist" << std::endl;
b2e465d6 711 continue;
b35d2f5f 712 }
b2e465d6
AL
713
714 // FindInCache is also expected to do an IMS check.
715 pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
716 if (File.end() == true)
c8e572e3
MV
717 {
718 if (Debug == true)
719 std::clog << "FindInCache returned end-Pointer" << std::endl;
b35d2f5f 720 return false;
c8e572e3 721 }
a52f938b 722
b2e465d6 723 Visited[File->ID] = true;
c8e572e3
MV
724 if (Debug == true)
725 std::clog << "with ID " << File->ID << " is valid" << std::endl;
b35d2f5f
AL
726 }
727
b2e465d6
AL
728 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
729 if (Visited[I] == false)
c8e572e3
MV
730 {
731 if (Debug == true)
732 std::clog << "File with ID" << I << " wasn't visited" << std::endl;
b2e465d6 733 return false;
c8e572e3 734 }
b35d2f5f 735
b35d2f5f
AL
736 if (_error->PendingError() == true)
737 {
c8e572e3
MV
738 if (Debug == true)
739 {
740 std::clog << "Validity failed because of pending errors:" << std::endl;
741 _error->DumpErrors();
742 }
b35d2f5f
AL
743 _error->Discard();
744 return false;
745 }
b35d2f5f 746
b2e465d6
AL
747 if (OutMap != 0)
748 *OutMap = Map.UnGuard();
b35d2f5f
AL
749 return true;
750}
751 /*}}}*/
b2e465d6 752// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 753// ---------------------------------------------------------------------
b2e465d6
AL
754/* Size is kind of an abstract notion that is only used for the progress
755 meter */
e7b470ee 756static unsigned long ComputeSize(FileIterator Start,FileIterator End)
b35d2f5f 757{
b2e465d6
AL
758 unsigned long TotalSize = 0;
759 for (; Start != End; Start++)
b35d2f5f 760 {
b2e465d6
AL
761 if ((*Start)->HasPackages() == false)
762 continue;
763 TotalSize += (*Start)->Size();
b35d2f5f 764 }
b2e465d6 765 return TotalSize;
2d11135a
AL
766}
767 /*}}}*/
b2e465d6 768// BuildCache - Merge the list of index files into the cache /*{{{*/
2d11135a 769// ---------------------------------------------------------------------
b2e465d6
AL
770/* */
771static bool BuildCache(pkgCacheGenerator &Gen,
772 OpProgress &Progress,
773 unsigned long &CurrentSize,unsigned long TotalSize,
e7b470ee 774 FileIterator Start, FileIterator End)
2d11135a 775{
45415543
AL
776 FileIterator I;
777 for (I = Start; I != End; I++)
2d11135a 778 {
45415543 779 if ((*I)->HasPackages() == false)
2d11135a
AL
780 continue;
781
45415543 782 if ((*I)->Exists() == false)
2d11135a 783 continue;
b2e465d6 784
45415543 785 if ((*I)->FindInCache(Gen.GetCache()).end() == false)
a77ad7c3
AL
786 {
787 _error->Warning("Duplicate sources.list entry %s",
45415543 788 (*I)->Describe().c_str());
a77ad7c3
AL
789 continue;
790 }
791
45415543 792 unsigned long Size = (*I)->Size();
db0db9fe 793 Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
b2e465d6 794 CurrentSize += Size;
2d11135a 795
45415543 796 if ((*I)->Merge(Gen,Progress) == false)
b2e465d6
AL
797 return false;
798 }
45415543
AL
799
800 if (Gen.HasFileDeps() == true)
801 {
802 Progress.Done();
803 TotalSize = ComputeSize(Start, End);
804 CurrentSize = 0;
805 for (I = Start; I != End; I++)
806 {
807 unsigned long Size = (*I)->Size();
808 Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
809 CurrentSize += Size;
810 if ((*I)->MergeFileProvides(Gen,Progress) == false)
811 return false;
812 }
813 }
2d11135a 814
b35d2f5f
AL
815 return true;
816}
817 /*}}}*/
b2e465d6 818// MakeStatusCache - Construct the status cache /*{{{*/
b35d2f5f 819// ---------------------------------------------------------------------
b2e465d6
AL
820/* This makes sure that the status cache (the cache that has all
821 index files from the sources list and all local ones) is ready
822 to be mmaped. If OutMap is not zero then a MMap object representing
823 the cache will be stored there. This is pretty much mandetory if you
824 are using AllowMem. AllowMem lets the function be run as non-root
825 where it builds the cache 'fast' into a memory buffer. */
826bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
827 MMap **OutMap,bool AllowMem)
b35d2f5f 828{
c8e572e3
MV
829 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
830 unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
67db871e 831
7db98ffc
MZ
832 vector<pkgIndexFile *> Files;
833 for (vector<metaIndex *>::const_iterator i = List.begin();
834 i != List.end();
835 i++)
836 {
837 vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
838 for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
839 j != Indexes->end();
840 j++)
841 Files.push_back (*j);
842 }
843
c8e572e3 844 unsigned long const EndOfSource = Files.size();
b2e465d6
AL
845 if (_system->AddStatusFiles(Files) == false)
846 return false;
c5f44afc 847
b2e465d6 848 // Decide if we can write to the files..
c8e572e3
MV
849 string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
850 string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
b2e465d6
AL
851
852 // Decide if we can write to the cache
853 bool Writeable = false;
854 if (CacheFile.empty() == false)
855 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
856 else
857 if (SrcCacheFile.empty() == false)
858 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
c8e572e3
MV
859 if (Debug == true)
860 std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
861
b2e465d6
AL
862 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
863 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
864
db0db9fe 865 Progress.OverallProgress(0,1,1,_("Reading package lists"));
b2e465d6
AL
866
867 // Cache is OK, Fin.
868 if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
869 {
db0db9fe 870 Progress.OverallProgress(1,1,1,_("Reading package lists"));
c8e572e3
MV
871 if (Debug == true)
872 std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
b2e465d6
AL
873 return true;
874 }
c8e572e3
MV
875 else if (Debug == true)
876 std::clog << "pkgcache.bin is NOT valid" << std::endl;
b2e465d6
AL
877
878 /* At this point we know we need to reconstruct the package cache,
879 begin. */
880 SPtr<FileFd> CacheF;
881 SPtr<DynamicMMap> Map;
882 if (Writeable == true && CacheFile.empty() == false)
883 {
884 unlink(CacheFile.c_str());
885 CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
7a3c2ab0 886 fchmod(CacheF->Fd(),0644);
b2e465d6 887 Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
b35d2f5f
AL
888 if (_error->PendingError() == true)
889 return false;
c8e572e3
MV
890 if (Debug == true)
891 std::clog << "Open filebased MMap" << std::endl;
b35d2f5f 892 }
b2e465d6 893 else
8ce4327b 894 {
b2e465d6 895 // Just build it in memory..
eb162ff7 896 Map = new DynamicMMap(0,MapSize);
c8e572e3
MV
897 if (Debug == true)
898 std::clog << "Open memory Map (not filebased)" << std::endl;
8ce4327b 899 }
b35d2f5f 900
b2e465d6 901 // Lets try the source cache.
b35d2f5f 902 unsigned long CurrentSize = 0;
b2e465d6
AL
903 unsigned long TotalSize = 0;
904 if (CheckValidity(SrcCacheFile,Files.begin(),
905 Files.begin()+EndOfSource) == true)
2d11135a 906 {
c8e572e3
MV
907 if (Debug == true)
908 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
b2e465d6
AL
909 // Preload the map with the source cache
910 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
c8e572e3 911 unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
eb162ff7
DK
912 if ((alloc == 0 && _error->PendingError())
913 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
914 SCacheF.Size()) == false)
b2e465d6
AL
915 return false;
916
917 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
c8e572e3 918
b2e465d6
AL
919 // Build the status cache
920 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 921 if (_error->PendingError() == true)
b2e465d6
AL
922 return false;
923 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
924 Files.begin()+EndOfSource,Files.end()) == false)
925 return false;
926 }
927 else
2d11135a 928 {
c8e572e3
MV
929 if (Debug == true)
930 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
b2e465d6 931 TotalSize = ComputeSize(Files.begin(),Files.end());
2d11135a 932
b2e465d6
AL
933 // Build the source cache
934 pkgCacheGenerator Gen(Map.Get(),&Progress);
935 if (_error->PendingError() == true)
936 return false;
937 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
938 Files.begin(),Files.begin()+EndOfSource) == false)
939 return false;
2d11135a 940
b2e465d6
AL
941 // Write it back
942 if (Writeable == true && SrcCacheFile.empty() == false)
2d11135a 943 {
b2e465d6
AL
944 FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
945 if (_error->PendingError() == true)
946 return false;
7a3c2ab0
AL
947
948 fchmod(SCacheF.Fd(),0644);
949
b2e465d6
AL
950 // Write out the main data
951 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
952 return _error->Error(_("IO Error saving source cache"));
953 SCacheF.Sync();
954
955 // Write out the proper header
956 Gen.GetCache().HeaderP->Dirty = false;
957 if (SCacheF.Seek(0) == false ||
958 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
959 return _error->Error(_("IO Error saving source cache"));
b2e465d6 960 Gen.GetCache().HeaderP->Dirty = true;
7a3c2ab0 961 SCacheF.Sync();
2d11135a
AL
962 }
963
b2e465d6
AL
964 // Build the status cache
965 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
966 Files.begin()+EndOfSource,Files.end()) == false)
967 return false;
2d11135a 968 }
c8e572e3
MV
969 if (Debug == true)
970 std::clog << "Caches are ready for shipping" << std::endl;
2d11135a 971
b2e465d6
AL
972 if (_error->PendingError() == true)
973 return false;
974 if (OutMap != 0)
2d11135a 975 {
b2e465d6 976 if (CacheF != 0)
2d11135a 977 {
b2e465d6 978 delete Map.UnGuard();
eb162ff7 979 *OutMap = new MMap(*CacheF,0);
2d11135a 980 }
b2e465d6
AL
981 else
982 {
983 *OutMap = Map.UnGuard();
984 }
2d11135a
AL
985 }
986
b2e465d6
AL
987 return true;
988}
989 /*}}}*/
990// MakeOnlyStatusCache - Build a cache with just the status files /*{{{*/
991// ---------------------------------------------------------------------
992/* */
993bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
994{
5177f802 995 unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
b2e465d6
AL
996 vector<pkgIndexFile *> Files;
997 unsigned long EndOfSource = Files.size();
998 if (_system->AddStatusFiles(Files) == false)
999 return false;
1000
eb162ff7 1001 SPtr<DynamicMMap> Map = new DynamicMMap(0,MapSize);
b2e465d6
AL
1002 unsigned long CurrentSize = 0;
1003 unsigned long TotalSize = 0;
1004
1005 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1006
1007 // Build the status cache
db0db9fe 1008 Progress.OverallProgress(0,1,1,_("Reading package lists"));
b2e465d6 1009 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 1010 if (_error->PendingError() == true)
b2e465d6
AL
1011 return false;
1012 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1013 Files.begin()+EndOfSource,Files.end()) == false)
1014 return false;
2d11135a 1015
b2e465d6
AL
1016 if (_error->PendingError() == true)
1017 return false;
1018 *OutMap = Map.UnGuard();
2d11135a 1019
b2e465d6 1020 return true;
2d11135a
AL
1021}
1022 /*}}}*/