]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
merge from lp:~mvo/apt/mvo
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
b2e465d6
AL
13#define APT_COMPATIBILITY 986
14
094a497d
AL
15#include <apt-pkg/pkgcachegen.h>
16#include <apt-pkg/error.h>
17#include <apt-pkg/version.h>
b35d2f5f
AL
18#include <apt-pkg/progress.h>
19#include <apt-pkg/sourcelist.h>
20#include <apt-pkg/configuration.h>
25396fb0 21#include <apt-pkg/aptconfiguration.h>
cdcc6d34 22#include <apt-pkg/strutl.h>
b2e465d6
AL
23#include <apt-pkg/sptr.h>
24#include <apt-pkg/pkgsystem.h>
aea7f4c8 25#include <apt-pkg/macros.h>
578bfd0a 26
afb1e2e3
MV
27#include <apt-pkg/tagfile.h>
28
b2e465d6 29#include <apti18n.h>
e7b470ee
AL
30
31#include <vector>
32
578bfd0a
AL
33#include <sys/stat.h>
34#include <unistd.h>
803fafcb 35#include <errno.h>
7ef72446 36#include <stdio.h>
578bfd0a 37 /*}}}*/
e7b470ee 38typedef vector<pkgIndexFile *>::iterator FileIterator;
d10cef82 39template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
578bfd0a
AL
40
41// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
42// ---------------------------------------------------------------------
25396fb0 43/* We set the dirty flag and make sure that is written to the disk */
b2e465d6 44pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45415543
AL
45 Map(*pMap), Cache(pMap,false), Progress(Prog),
46 FoundFileDeps(0)
578bfd0a 47{
ddc1d8d0 48 CurrentFile = 0;
b2e465d6 49 memset(UniqHash,0,sizeof(UniqHash));
ddc1d8d0 50
578bfd0a
AL
51 if (_error->PendingError() == true)
52 return;
b2e465d6 53
578bfd0a
AL
54 if (Map.Size() == 0)
55 {
b2e465d6
AL
56 // Setup the map interface..
57 Cache.HeaderP = (pkgCache::Header *)Map.Data();
c5f44afc
DK
58 if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
59 return;
60
b2e465d6 61 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
c5f44afc 62
b2e465d6 63 // Starting header
578bfd0a 64 *Cache.HeaderP = pkgCache::Header();
a9fe5928
DK
65 map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
66 Cache.HeaderP->VerSysName = idxVerSysName;
67 map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
68 Cache.HeaderP->Architecture = idxArchitecture;
69 if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
70 return;
c5f44afc 71 Cache.ReMap();
578bfd0a 72 }
b2e465d6
AL
73 else
74 {
75 // Map directly from the existing file
76 Cache.ReMap();
77 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
78 if (Cache.VS != _system->VS)
79 {
80 _error->Error(_("Cache has an incompatible versioning system"));
81 return;
82 }
83 }
84
578bfd0a
AL
85 Cache.HeaderP->Dirty = true;
86 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
87}
88 /*}}}*/
89// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
90// ---------------------------------------------------------------------
91/* We sync the data then unset the dirty flag in two steps so as to
92 advoid a problem during a crash */
93pkgCacheGenerator::~pkgCacheGenerator()
94{
95 if (_error->PendingError() == true)
96 return;
97 if (Map.Sync() == false)
98 return;
99
100 Cache.HeaderP->Dirty = false;
101 Map.Sync(0,sizeof(pkgCache::Header));
102}
103 /*}}}*/
a9fe5928
DK
104void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
105 if (oldMap == newMap)
106 return;
107
108 Cache.ReMap(false);
109
110 CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
111
112 for (size_t i = 0; i < _count(UniqHash); ++i)
113 if (UniqHash[i] != 0)
114 UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
115
7635093c 116 for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
a9fe5928 117 i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
f7a35f2e 118 (*i)->ReMap(oldMap, newMap);
7635093c 119 for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
a9fe5928 120 i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
f7a35f2e 121 (*i)->ReMap(oldMap, newMap);
7635093c 122 for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
a9fe5928 123 i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
f7a35f2e 124 (*i)->ReMap(oldMap, newMap);
7635093c 125 for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
a9fe5928 126 i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
f7a35f2e 127 (*i)->ReMap(oldMap, newMap);
7635093c 128 for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
a9fe5928 129 i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
f7a35f2e 130 (*i)->ReMap(oldMap, newMap);
7635093c 131 for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
a9fe5928 132 i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
f7a35f2e 133 (*i)->ReMap(oldMap, newMap);
7635093c 134 for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
a9fe5928 135 i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
f7a35f2e 136 (*i)->ReMap(oldMap, newMap);
a9fe5928 137} /*}}}*/
7e58ab0c 138// CacheGenerator::WriteStringInMap /*{{{*/
a9fe5928 139map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
7e58ab0c 140 const unsigned long &Len) {
a9fe5928
DK
141 void const * const oldMap = Map.Data();
142 map_ptrloc const index = Map.WriteString(String, Len);
143 if (index != 0)
144 ReMap(oldMap, Map.Data());
145 return index;
7e58ab0c
DK
146}
147 /*}}}*/
148// CacheGenerator::WriteStringInMap /*{{{*/
a9fe5928
DK
149map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
150 void const * const oldMap = Map.Data();
151 map_ptrloc const index = Map.WriteString(String);
152 if (index != 0)
153 ReMap(oldMap, Map.Data());
154 return index;
7e58ab0c
DK
155}
156 /*}}}*/
a9fe5928
DK
157map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
158 void const * const oldMap = Map.Data();
159 map_ptrloc const index = Map.Allocate(size);
160 if (index != 0)
161 ReMap(oldMap, Map.Data());
162 return index;
7e58ab0c
DK
163}
164 /*}}}*/
578bfd0a
AL
165// CacheGenerator::MergeList - Merge the package list /*{{{*/
166// ---------------------------------------------------------------------
167/* This provides the generation of the entries in the cache. Each loop
168 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
169bool pkgCacheGenerator::MergeList(ListParser &List,
170 pkgCache::VerIterator *OutVer)
578bfd0a
AL
171{
172 List.Owner = this;
0149949b 173
f9eec0e7 174 unsigned int Counter = 0;
0149949b 175 while (List.Step() == true)
578bfd0a 176 {
5bf15716 177 string const PackageName = List.Package();
65a1e968
AL
178 if (PackageName.empty() == true)
179 return false;
5bf15716 180
566046f4 181 string const Arch = List.Architecture();
28166356 182
8b32e920 183 // Get a pointer to the package structure
9ddf7030 184 pkgCache::PkgIterator Pkg;
a9fe5928 185 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
28166356 186 if (NewPackage(Pkg, PackageName, Arch) == false)
6804503b 187 return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
a246f2dc 188 Counter++;
ddc1d8d0
AL
189 if (Counter % 100 == 0 && Progress != 0)
190 Progress->Progress(List.Offset());
8ce4327b 191
578bfd0a
AL
192 /* Get a pointer to the version structure. We know the list is sorted
193 so we use that fact in the search. Insertion of new versions is
194 done with correct sorting */
195 string Version = List.Version();
f55a958f
AL
196 if (Version.empty() == true)
197 {
770c32ec
MV
198 // we first process the package, then the descriptions
199 // (this has the bonus that we get MMap error when we run out
200 // of MMap space)
32b9a14c 201 pkgCache::VerIterator Ver(Cache);
a9fe5928 202 Dynamic<pkgCache::VerIterator> DynVer(Ver);
32b9a14c 203 if (List.UsePackage(Pkg, Ver) == false)
6804503b 204 return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
7a3c2ab0 205 PackageName.c_str());
770c32ec 206
a52f938b
OS
207 // Find the right version to write the description
208 MD5SumValue CurMd5 = List.Description_md5();
32b9a14c 209 Ver = Pkg.VersionList();
770c32ec 210
a9fe5928 211 for (; Ver.end() == false; ++Ver)
a52f938b
OS
212 {
213 pkgCache::DescIterator Desc = Ver.DescriptionList();
a9fe5928 214 Dynamic<pkgCache::DescIterator> DynDesc(Desc);
a52f938b 215 map_ptrloc *LastDesc = &Ver->DescriptionList;
c76c44b1 216 bool duplicate=false;
217
218 // don't add a new description if we have one for the given
219 // md5 && language
9ee47c29 220 for ( ; Desc.end() == false; Desc++)
c76c44b1 221 if (MD5SumValue(Desc.md5()) == CurMd5 &&
222 Desc.LanguageCode() == List.DescriptionLanguage())
223 duplicate=true;
224 if(duplicate)
225 continue;
226
227 for (Desc = Ver.DescriptionList();
9ee47c29 228 Desc.end() == false;
c76c44b1 229 LastDesc = &Desc->NextDesc, Desc++)
770c32ec 230 {
770c32ec
MV
231 if (MD5SumValue(Desc.md5()) == CurMd5)
232 {
a52f938b 233 // Add new description
a9fe5928
DK
234 void const * const oldMap = Map.Data();
235 map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
236 if (oldMap != Map.Data())
237 LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
238 *LastDesc = descindex;
a52f938b 239 Desc->ParentPkg = Pkg.Index();
770c32ec 240
c5f44afc 241 if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
f0f66a3d 242 return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
a52f938b
OS
243 break;
244 }
770c32ec 245 }
a52f938b 246 }
770c32ec 247
f55a958f
AL
248 continue;
249 }
250
578bfd0a 251 pkgCache::VerIterator Ver = Pkg.VersionList();
a9fe5928 252 Dynamic<pkgCache::VerIterator> DynVer(Ver);
a52f938b 253 map_ptrloc *LastVer = &Pkg->VersionList;
a9fe5928 254 void const * oldMap = Map.Data();
2246928b 255 int Res = 1;
7e2b56a3 256 unsigned long const Hash = List.VersionHash();
9ee47c29 257 for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
578bfd0a 258 {
c24972cb 259 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
7e2b56a3
DK
260 // Version is higher as current version - insert here
261 if (Res > 0)
578bfd0a 262 break;
7e2b56a3
DK
263 // Versionstrings are equal - is hash also equal?
264 if (Res == 0 && Ver->Hash == Hash)
265 break;
266 // proceed with the next till we have either the right
267 // or we found another version (which will be lower)
578bfd0a 268 }
7e2b56a3
DK
269
270 /* We already have a version for this item, record that we saw it */
e426a5ff 271 if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
578bfd0a 272 {
f55a958f 273 if (List.UsePackage(Pkg,Ver) == false)
6804503b 274 return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
7a3c2ab0 275 PackageName.c_str());
f78439bf 276
578bfd0a 277 if (NewFileVer(Ver,List) == false)
6804503b 278 return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
7a3c2ab0 279 PackageName.c_str());
578bfd0a 280
ddc1d8d0
AL
281 // Read only a single record and return
282 if (OutVer != 0)
283 {
284 *OutVer = Ver;
45415543 285 FoundFileDeps |= List.HasFileDeps();
ddc1d8d0
AL
286 return true;
287 }
288
578bfd0a 289 continue;
204fbdcc
AL
290 }
291
578bfd0a 292 // Add a new version
a9fe5928
DK
293 map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
294 if (verindex == 0 && _error->PendingError())
295 return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
296 PackageName.c_str(), 1);
297
298 if (oldMap != Map.Data())
299 LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
300 *LastVer = verindex;
f55a958f 301 Ver->ParentPkg = Pkg.Index();
204fbdcc 302 Ver->Hash = Hash;
a52f938b 303
a9fe5928
DK
304 if (List.NewVersion(Ver) == false)
305 return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
306 PackageName.c_str(), 2);
0149949b 307
f55a958f 308 if (List.UsePackage(Pkg,Ver) == false)
6804503b 309 return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
7a3c2ab0 310 PackageName.c_str());
f55a958f 311
578bfd0a 312 if (NewFileVer(Ver,List) == false)
a9fe5928
DK
313 return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
314 PackageName.c_str(), 3);
ddc1d8d0
AL
315
316 // Read only a single record and return
317 if (OutVer != 0)
318 {
319 *OutVer = Ver;
45415543 320 FoundFileDeps |= List.HasFileDeps();
ddc1d8d0
AL
321 return true;
322 }
a52f938b
OS
323
324 /* Record the Description data. Description data always exist in
325 Packages and Translation-* files. */
326 pkgCache::DescIterator Desc = Ver.DescriptionList();
a9fe5928 327 Dynamic<pkgCache::DescIterator> DynDesc(Desc);
a52f938b 328 map_ptrloc *LastDesc = &Ver->DescriptionList;
a9fe5928 329
a52f938b 330 // Skip to the end of description set
9ee47c29 331 for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
a52f938b
OS
332
333 // Add new description
a9fe5928
DK
334 oldMap = Map.Data();
335 map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
336 if (oldMap != Map.Data())
337 LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
338 *LastDesc = descindex;
a52f938b
OS
339 Desc->ParentPkg = Pkg.Index();
340
c5f44afc 341 if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
f0f66a3d 342 return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
578bfd0a 343 }
0149949b 344
45415543
AL
345 FoundFileDeps |= List.HasFileDeps();
346
6a3da7a6
AL
347 if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
348 return _error->Error(_("Wow, you exceeded the number of package "
349 "names this APT is capable of."));
350 if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
351 return _error->Error(_("Wow, you exceeded the number of versions "
352 "this APT is capable of."));
a52f938b
OS
353 if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
354 return _error->Error(_("Wow, you exceeded the number of descriptions "
355 "this APT is capable of."));
6a3da7a6
AL
356 if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
357 return _error->Error(_("Wow, you exceeded the number of dependencies "
358 "this APT is capable of."));
578bfd0a
AL
359 return true;
360}
361 /*}}}*/
45415543
AL
362// CacheGenerator::MergeFileProvides - Merge file provides /*{{{*/
363// ---------------------------------------------------------------------
364/* If we found any file depends while parsing the main list we need to
365 resolve them. Since it is undesired to load the entire list of files
366 into the cache as virtual packages we do a two stage effort. MergeList
367 identifies the file depends and this creates Provdies for them by
368 re-parsing all the indexs. */
369bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
370{
371 List.Owner = this;
372
373 unsigned int Counter = 0;
374 while (List.Step() == true)
375 {
376 string PackageName = List.Package();
377 if (PackageName.empty() == true)
378 return false;
379 string Version = List.Version();
380 if (Version.empty() == true)
381 continue;
382
383 pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
a9fe5928 384 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
45415543 385 if (Pkg.end() == true)
6804503b 386 return _error->Error(_("Error occurred while processing %s (FindPkg)"),
45415543
AL
387 PackageName.c_str());
388 Counter++;
389 if (Counter % 100 == 0 && Progress != 0)
390 Progress->Progress(List.Offset());
391
392 unsigned long Hash = List.VersionHash();
393 pkgCache::VerIterator Ver = Pkg.VersionList();
a9fe5928 394 Dynamic<pkgCache::VerIterator> DynVer(Ver);
9ee47c29 395 for (; Ver.end() == false; Ver++)
45415543
AL
396 {
397 if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
398 {
399 if (List.CollectFileProvides(Cache,Ver) == false)
6804503b 400 return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
45415543
AL
401 break;
402 }
403 }
404
405 if (Ver.end() == true)
406 _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
407 }
408
409 return true;
410}
411 /*}}}*/
5bf15716
DK
412// CacheGenerator::NewGroup - Add a new group /*{{{*/
413// ---------------------------------------------------------------------
414/* This creates a new group structure and adds it to the hash table */
33dd02e3
DK
415bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
416{
417 Grp = Cache.FindGrp(Name);
418 if (Grp.end() == false)
419 return true;
5bf15716 420
33dd02e3 421 // Get a structure
a9fe5928 422 map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
33dd02e3
DK
423 if (unlikely(Group == 0))
424 return false;
5bf15716 425
33dd02e3 426 Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
a9fe5928
DK
427 map_ptrloc const idxName = WriteStringInMap(Name);
428 if (unlikely(idxName == 0))
33dd02e3 429 return false;
a9fe5928 430 Grp->Name = idxName;
5bf15716 431
33dd02e3
DK
432 // Insert it into the hash table
433 unsigned long const Hash = Cache.Hash(Name);
434 Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
435 Cache.HeaderP->GrpHashTable[Hash] = Group;
5bf15716 436
52c41485 437 Grp->ID = Cache.HeaderP->GroupCount++;
33dd02e3 438 return true;
5bf15716
DK
439}
440 /*}}}*/
578bfd0a
AL
441// CacheGenerator::NewPackage - Add a new package /*{{{*/
442// ---------------------------------------------------------------------
443/* This creates a new package structure and adds it to the hash table */
5bf15716
DK
444bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
445 const string &Arch) {
446 pkgCache::GrpIterator Grp;
a9fe5928 447 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
5bf15716
DK
448 if (unlikely(NewGroup(Grp, Name) == false))
449 return false;
450
451 Pkg = Grp.FindPkg(Arch);
452 if (Pkg.end() == false)
453 return true;
a52f938b 454
578bfd0a 455 // Get a structure
a9fe5928 456 map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
5bf15716 457 if (unlikely(Package == 0))
578bfd0a 458 return false;
f55a958f 459 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
5bf15716 460
c408e01e
DK
461 // Insert the package into our package list
462 if (Grp->FirstPackage == 0) // the group is new
463 {
464 // Insert it into the hash table
465 unsigned long const Hash = Cache.Hash(Name);
466 Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
467 Cache.HeaderP->PkgHashTable[Hash] = Package;
468 Grp->FirstPackage = Package;
469 }
470 else // Group the Packages together
471 {
472 // this package is the new last package
473 pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
474 Pkg->NextPackage = LastPkg->NextPackage;
475 LastPkg->NextPackage = Package;
476 }
477 Grp->LastPackage = Package;
5bf15716
DK
478
479 // Set the name, arch and the ID
480 Pkg->Name = Grp->Name;
481 Pkg->Group = Grp.Index();
959470da
DK
482 // all is mapped to the native architecture
483 map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
a9fe5928 484 if (unlikely(idxArch == 0))
578bfd0a 485 return false;
a9fe5928 486 Pkg->Arch = idxArch;
578bfd0a 487 Pkg->ID = Cache.HeaderP->PackageCount++;
5bf15716 488
578bfd0a
AL
489 return true;
490}
491 /*}}}*/
492// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
493// ---------------------------------------------------------------------
494/* */
f55a958f 495bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
496 ListParser &List)
497{
ddc1d8d0
AL
498 if (CurrentFile == 0)
499 return true;
500
dcb79bae 501 // Get a structure
a9fe5928 502 map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
dcb79bae
AL
503 if (VerFile == 0)
504 return 0;
505
506 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
507 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
508
509 // Link it to the end of the list
349cd3b8 510 map_ptrloc *Last = &Ver->FileList;
eb162ff7 511 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
03e39e59
AL
512 Last = &V->NextFile;
513 VF->NextFile = *Last;
514 *Last = VF.Index();
515
dcb79bae
AL
516 VF->Offset = List.Offset();
517 VF->Size = List.Size();
ad00ae81
AL
518 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
519 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
520 Cache.HeaderP->VerFileCount++;
521
f55a958f 522 return true;
578bfd0a
AL
523}
524 /*}}}*/
525// CacheGenerator::NewVersion - Create a new Version /*{{{*/
526// ---------------------------------------------------------------------
f55a958f 527/* This puts a version structure in the linked list */
578bfd0a 528unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
171c75f1 529 const string &VerStr,
578bfd0a
AL
530 unsigned long Next)
531{
f55a958f 532 // Get a structure
a9fe5928 533 map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
f55a958f 534 if (Version == 0)
0149949b 535 return 0;
f55a958f
AL
536
537 // Fill it in
538 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
f55a958f
AL
539 Ver->NextVer = Next;
540 Ver->ID = Cache.HeaderP->VersionCount++;
a9fe5928
DK
541 map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
542 if (unlikely(idxVerStr == 0))
0149949b 543 return 0;
a9fe5928 544 Ver->VerStr = idxVerStr;
f55a958f 545
0149949b 546 return Version;
578bfd0a
AL
547}
548 /*}}}*/
a52f938b
OS
549// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
550// ---------------------------------------------------------------------
551/* */
552bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
553 ListParser &List)
554{
555 if (CurrentFile == 0)
556 return true;
557
558 // Get a structure
a9fe5928 559 map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
a52f938b 560 if (DescFile == 0)
c5f44afc 561 return false;
770c32ec 562
a52f938b
OS
563 pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
564 DF->File = CurrentFile - Cache.PkgFileP;
770c32ec 565
a52f938b
OS
566 // Link it to the end of the list
567 map_ptrloc *Last = &Desc->FileList;
eb162ff7 568 for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
a52f938b 569 Last = &D->NextFile;
770c32ec 570
a52f938b
OS
571 DF->NextFile = *Last;
572 *Last = DF.Index();
573
574 DF->Offset = List.Offset();
575 DF->Size = List.Size();
576 if (Cache.HeaderP->MaxDescFileSize < DF->Size)
577 Cache.HeaderP->MaxDescFileSize = DF->Size;
578 Cache.HeaderP->DescFileCount++;
579
580 return true;
581}
582 /*}}}*/
583// CacheGenerator::NewDescription - Create a new Description /*{{{*/
584// ---------------------------------------------------------------------
585/* This puts a description structure in the linked list */
586map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
c76c44b1 587 const string &Lang,
588 const MD5SumValue &md5sum,
a52f938b
OS
589 map_ptrloc Next)
590{
591 // Get a structure
a9fe5928 592 map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
a52f938b
OS
593 if (Description == 0)
594 return 0;
595
596 // Fill it in
597 Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
598 Desc->NextDesc = Next;
599 Desc->ID = Cache.HeaderP->DescriptionCount++;
a9fe5928
DK
600 map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
601 map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
602 if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
c5f44afc 603 return 0;
a9fe5928
DK
604 Desc->language_code = idxlanguage_code;
605 Desc->md5sum = idxmd5sum;
a52f938b
OS
606
607 return Description;
608}
609 /*}}}*/
25396fb0
DK
610// CacheGenerator::FinishCache - do various finish operations /*{{{*/
611// ---------------------------------------------------------------------
612/* This prepares the Cache for delivery */
2e5f4e45 613bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
33dd02e3
DK
614{
615 // FIXME: add progress reporting for this operation
616 // Do we have different architectures in your groups ?
617 vector<string> archs = APT::Configuration::getArchitectures();
618 if (archs.size() > 1)
619 {
620 // Create Conflicts in between the group
a9fe5928
DK
621 pkgCache::GrpIterator G = GetCache().GrpBegin();
622 Dynamic<pkgCache::GrpIterator> DynG(G);
623 for (; G.end() != true; G++)
33dd02e3
DK
624 {
625 string const PkgName = G.Name();
a9fe5928
DK
626 pkgCache::PkgIterator P = G.PackageList();
627 Dynamic<pkgCache::PkgIterator> DynP(P);
628 for (; P.end() != true; P = G.NextPkg(P))
33dd02e3 629 {
33dd02e3 630 pkgCache::PkgIterator allPkg;
a9fe5928
DK
631 Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
632 pkgCache::VerIterator V = P.VersionList();
633 Dynamic<pkgCache::VerIterator> DynV(V);
634 for (; V.end() != true; V++)
33dd02e3 635 {
a1ac2ca8 636 char const * const Arch = P.Arch();
33dd02e3
DK
637 map_ptrloc *OldDepLast = NULL;
638 /* MultiArch handling introduces a lot of implicit Dependencies:
639 - MultiArch: same → Co-Installable if they have the same version
640 - Architecture: all → Need to be Co-Installable for internal reasons
641 - All others conflict with all other group members */
ca238ede 642 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
33dd02e3
DK
643 for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
644 {
645 if (*A == Arch)
646 continue;
647 /* We allow only one installed arch at the time
648 per group, therefore each group member conflicts
649 with all other group members */
650 pkgCache::PkgIterator D = G.FindPkg(*A);
a9fe5928 651 Dynamic<pkgCache::PkgIterator> DynD(D);
33dd02e3
DK
652 if (D.end() == true)
653 continue;
654 if (coInstall == true)
655 {
656 // Replaces: ${self}:other ( << ${binary:Version})
657 NewDepends(D, V, V.VerStr(),
658 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
659 OldDepLast);
660 // Breaks: ${self}:other (!= ${binary:Version})
661 NewDepends(D, V, V.VerStr(),
3b527295 662 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
33dd02e3 663 OldDepLast);
33dd02e3
DK
664 } else {
665 // Conflicts: ${self}:other
28166356
DK
666 NewDepends(D, V, "",
667 pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
668 OldDepLast);
33dd02e3
DK
669 }
670 }
671 }
672 }
673 }
674 }
675 return true;
25396fb0
DK
676}
677 /*}}}*/
678// CacheGenerator::NewDepends - Create a dependency element /*{{{*/
dcb79bae
AL
679// ---------------------------------------------------------------------
680/* This creates a dependency element in the tree. It is linked to the
681 version and to the package that it is pointing to. */
25396fb0
DK
682bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
683 pkgCache::VerIterator &Ver,
684 string const &Version,
685 unsigned int const &Op,
686 unsigned int const &Type,
687 map_ptrloc *OldDepLast)
dcb79bae 688{
a9fe5928 689 void const * const oldMap = Map.Data();
dcb79bae 690 // Get a structure
a9fe5928 691 map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
5bf15716 692 if (unlikely(Dependency == 0))
dcb79bae
AL
693 return false;
694
695 // Fill it in
696 pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
a9fe5928 697 Dynamic<pkgCache::DepIterator> DynDep(Dep);
dcb79bae
AL
698 Dep->ParentVer = Ver.Index();
699 Dep->Type = Type;
700 Dep->CompareOp = Op;
701 Dep->ID = Cache.HeaderP->DependsCount++;
5bf15716 702
dcb79bae
AL
703 // Probe the reverse dependency list for a version string that matches
704 if (Version.empty() == false)
705 {
b2e465d6 706/* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
dcb79bae 707 if (I->Version != 0 && I.TargetVer() == Version)
f9eec0e7 708 Dep->Version = I->Version;*/
a9fe5928
DK
709 if (Dep->Version == 0) {
710 map_ptrloc const index = WriteStringInMap(Version);
711 if (unlikely(index == 0))
dcb79bae 712 return false;
a9fe5928
DK
713 Dep->Version = index;
714 }
dcb79bae 715 }
25396fb0 716
dcb79bae
AL
717 // Link it to the package
718 Dep->Package = Pkg.Index();
719 Dep->NextRevDepends = Pkg->RevDepends;
720 Pkg->RevDepends = Dep.Index();
25396fb0
DK
721
722 // Do we know where to link the Dependency to?
723 if (OldDepLast == NULL)
c1a22377 724 {
f9eec0e7 725 OldDepLast = &Ver->DependsList;
eb162ff7 726 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
f9eec0e7 727 OldDepLast = &D->NextDepends;
a9fe5928
DK
728 } else if (oldMap != Map.Data())
729 OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
45415543 730
f9eec0e7
AL
731 Dep->NextDepends = *OldDepLast;
732 *OldDepLast = Dep.Index();
733 OldDepLast = &Dep->NextDepends;
c1a22377 734
dcb79bae
AL
735 return true;
736}
737 /*}}}*/
25396fb0
DK
738// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
739// ---------------------------------------------------------------------
740/* This creates a Group and the Package to link this dependency to if
741 needed and handles also the caching of the old endpoint */
32b9a14c 742bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
25396fb0
DK
743 const string &PackageName,
744 const string &Arch,
745 const string &Version,
746 unsigned int Op,
747 unsigned int Type)
748{
749 pkgCache::GrpIterator Grp;
a9fe5928 750 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
25396fb0
DK
751 if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
752 return false;
753
754 // Locate the target package
755 pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
a9fe5928 756 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
25396fb0
DK
757 if (Pkg.end() == true) {
758 if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
759 return false;
760 }
761
762 // Is it a file dependency?
763 if (unlikely(PackageName[0] == '/'))
764 FoundFileDeps = true;
765
766 /* Caching the old end point speeds up generation substantially */
767 if (OldDepVer != Ver) {
768 OldDepLast = NULL;
769 OldDepVer = Ver;
770 }
771
772 return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
773}
774 /*}}}*/
dcb79bae
AL
775// ListParser::NewProvides - Create a Provides element /*{{{*/
776// ---------------------------------------------------------------------
777/* */
32b9a14c 778bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
67e0766f
DK
779 const string &PkgName,
780 const string &PkgArch,
171c75f1 781 const string &Version)
dcb79bae
AL
782{
783 pkgCache &Cache = Owner->Cache;
8efa2a3b
AL
784
785 // We do not add self referencing provides
566046f4 786 if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
959470da 787 (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
8efa2a3b 788 return true;
dcb79bae
AL
789
790 // Get a structure
a9fe5928 791 map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
5bf15716 792 if (unlikely(Provides == 0))
dcb79bae 793 return false;
a7e66b17 794 Cache.HeaderP->ProvidesCount++;
dcb79bae
AL
795
796 // Fill it in
797 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
a9fe5928 798 Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
dcb79bae
AL
799 Prv->Version = Ver.Index();
800 Prv->NextPkgProv = Ver->ProvidesList;
801 Ver->ProvidesList = Prv.Index();
5bf15716 802 if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
dcb79bae
AL
803 return false;
804
805 // Locate the target package
8efa2a3b 806 pkgCache::PkgIterator Pkg;
a9fe5928 807 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
67e0766f 808 if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
8efa2a3b 809 return false;
dcb79bae
AL
810
811 // Link it to the package
812 Prv->ParentPkg = Pkg.Index();
813 Prv->NextProvides = Pkg->ProvidesList;
814 Pkg->ProvidesList = Prv.Index();
815
816 return true;
817}
818 /*}}}*/
578bfd0a
AL
819// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
820// ---------------------------------------------------------------------
821/* This is used to select which file is to be associated with all newly
b2e465d6 822 added versions. The caller is responsible for setting the IMS fields. */
171c75f1 823bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
b2e465d6
AL
824 const pkgIndexFile &Index,
825 unsigned long Flags)
578bfd0a 826{
578bfd0a 827 // Get some space for the structure
a9fe5928
DK
828 map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
829 if (unlikely(idxFile == 0))
578bfd0a 830 return false;
a9fe5928
DK
831 CurrentFile = Cache.PkgFileP + idxFile;
832
578bfd0a 833 // Fill it in
a9fe5928
DK
834 map_ptrloc const idxFileName = WriteStringInMap(File);
835 map_ptrloc const idxSite = WriteUniqString(Site);
836 if (unlikely(idxFileName == 0 || idxSite == 0))
837 return false;
838 CurrentFile->FileName = idxFileName;
839 CurrentFile->Site = idxSite;
578bfd0a
AL
840 CurrentFile->NextFile = Cache.HeaderP->FileList;
841 CurrentFile->Flags = Flags;
e1b74f61 842 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
a9fe5928
DK
843 map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
844 if (unlikely(idxIndexType == 0))
845 return false;
846 CurrentFile->IndexType = idxIndexType;
578bfd0a 847 PkgFileName = File;
ad00ae81 848 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 849 Cache.HeaderP->PackageFileCount++;
b2e465d6 850
ddc1d8d0 851 if (Progress != 0)
b2e465d6 852 Progress->SubProgress(Index.Size());
8efa2a3b 853 return true;
578bfd0a
AL
854}
855 /*}}}*/
f55a958f
AL
856// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
857// ---------------------------------------------------------------------
858/* This is used to create handles to strings. Given the same text it
859 always returns the same number */
860unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
861 unsigned int Size)
862{
f9eec0e7
AL
863 /* We use a very small transient hash table here, this speeds up generation
864 by a fair amount on slower machines */
865 pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
866 if (Bucket != 0 &&
867 stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
868 return Bucket->String;
869
f55a958f
AL
870 // Search for an insertion point
871 pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
872 int Res = 1;
349cd3b8 873 map_ptrloc *Last = &Cache.HeaderP->StringList;
f55a958f
AL
874 for (; I != Cache.StringItemP; Last = &I->NextItem,
875 I = Cache.StringItemP + I->NextItem)
876 {
9c14e3d6 877 Res = stringcmp(S,S+Size,Cache.StrP + I->String);
f55a958f
AL
878 if (Res >= 0)
879 break;
880 }
881
882 // Match
883 if (Res == 0)
f9eec0e7
AL
884 {
885 Bucket = I;
0149949b 886 return I->String;
f9eec0e7 887 }
f55a958f
AL
888
889 // Get a structure
a9fe5928
DK
890 void const * const oldMap = Map.Data();
891 map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
f55a958f 892 if (Item == 0)
0149949b
AL
893 return 0;
894
a9fe5928
DK
895 map_ptrloc const idxString = WriteStringInMap(S,Size);
896 if (unlikely(idxString == 0))
897 return 0;
898 if (oldMap != Map.Data()) {
899 Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
900 I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
901 }
902 *Last = Item;
903
f55a958f
AL
904 // Fill in the structure
905 pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
906 ItemP->NextItem = I - Cache.StringItemP;
a9fe5928
DK
907 ItemP->String = idxString;
908
f9eec0e7 909 Bucket = ItemP;
0149949b 910 return ItemP->String;
f55a958f
AL
911}
912 /*}}}*/
b2e465d6 913// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 914// ---------------------------------------------------------------------
b2e465d6
AL
915/* This just verifies that each file in the list of index files exists,
916 has matching attributes with the cache and the cache does not have
917 any extra files. */
171c75f1 918static bool CheckValidity(const string &CacheFile, FileIterator Start,
e7b470ee 919 FileIterator End,MMap **OutMap = 0)
b35d2f5f 920{
c8e572e3 921 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
b2e465d6
AL
922 // No file, certainly invalid
923 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
c8e572e3
MV
924 {
925 if (Debug == true)
926 std::clog << "CacheFile doesn't exist" << std::endl;
b35d2f5f 927 return false;
c8e572e3
MV
928 }
929
b2e465d6 930 // Map it
b35d2f5f 931 FileFd CacheF(CacheFile,FileFd::ReadOnly);
eb162ff7 932 SPtr<MMap> Map = new MMap(CacheF,0);
b35d2f5f 933 pkgCache Cache(Map);
b2e465d6 934 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f 935 {
c8e572e3
MV
936 if (Debug == true)
937 std::clog << "Errors are pending or Map is empty()" << std::endl;
b35d2f5f
AL
938 _error->Discard();
939 return false;
940 }
b35d2f5f 941
b2e465d6
AL
942 /* Now we check every index file, see if it is in the cache,
943 verify the IMS data and check that it is on the disk too.. */
944 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
945 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
946 for (; Start != End; Start++)
c8e572e3
MV
947 {
948 if (Debug == true)
949 std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
b2e465d6 950 if ((*Start)->HasPackages() == false)
c8e572e3
MV
951 {
952 if (Debug == true)
953 std::clog << "Has NO packages" << std::endl;
b2e465d6 954 continue;
c8e572e3 955 }
a77ad7c3 956
b2e465d6 957 if ((*Start)->Exists() == false)
b35d2f5f 958 {
a791a450 959#if 0 // mvo: we no longer give a message here (Default Sources spec)
b2e465d6
AL
960 _error->WarningE("stat",_("Couldn't stat source package list %s"),
961 (*Start)->Describe().c_str());
a791a450 962#endif
c8e572e3
MV
963 if (Debug == true)
964 std::clog << "file doesn't exist" << std::endl;
b2e465d6 965 continue;
b35d2f5f 966 }
b2e465d6
AL
967
968 // FindInCache is also expected to do an IMS check.
969 pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
970 if (File.end() == true)
c8e572e3
MV
971 {
972 if (Debug == true)
973 std::clog << "FindInCache returned end-Pointer" << std::endl;
b35d2f5f 974 return false;
c8e572e3 975 }
a52f938b 976
b2e465d6 977 Visited[File->ID] = true;
c8e572e3
MV
978 if (Debug == true)
979 std::clog << "with ID " << File->ID << " is valid" << std::endl;
b35d2f5f
AL
980 }
981
b2e465d6
AL
982 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
983 if (Visited[I] == false)
c8e572e3
MV
984 {
985 if (Debug == true)
986 std::clog << "File with ID" << I << " wasn't visited" << std::endl;
b2e465d6 987 return false;
c8e572e3 988 }
b35d2f5f 989
b35d2f5f
AL
990 if (_error->PendingError() == true)
991 {
c8e572e3
MV
992 if (Debug == true)
993 {
994 std::clog << "Validity failed because of pending errors:" << std::endl;
995 _error->DumpErrors();
996 }
b35d2f5f
AL
997 _error->Discard();
998 return false;
999 }
b35d2f5f 1000
b2e465d6
AL
1001 if (OutMap != 0)
1002 *OutMap = Map.UnGuard();
b35d2f5f
AL
1003 return true;
1004}
1005 /*}}}*/
b2e465d6 1006// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 1007// ---------------------------------------------------------------------
b2e465d6
AL
1008/* Size is kind of an abstract notion that is only used for the progress
1009 meter */
e7b470ee 1010static unsigned long ComputeSize(FileIterator Start,FileIterator End)
b35d2f5f 1011{
b2e465d6
AL
1012 unsigned long TotalSize = 0;
1013 for (; Start != End; Start++)
b35d2f5f 1014 {
b2e465d6
AL
1015 if ((*Start)->HasPackages() == false)
1016 continue;
1017 TotalSize += (*Start)->Size();
b35d2f5f 1018 }
b2e465d6 1019 return TotalSize;
2d11135a
AL
1020}
1021 /*}}}*/
b2e465d6 1022// BuildCache - Merge the list of index files into the cache /*{{{*/
2d11135a 1023// ---------------------------------------------------------------------
b2e465d6
AL
1024/* */
1025static bool BuildCache(pkgCacheGenerator &Gen,
2e5f4e45 1026 OpProgress *Progress,
b2e465d6 1027 unsigned long &CurrentSize,unsigned long TotalSize,
e7b470ee 1028 FileIterator Start, FileIterator End)
2d11135a 1029{
45415543
AL
1030 FileIterator I;
1031 for (I = Start; I != End; I++)
2d11135a 1032 {
45415543 1033 if ((*I)->HasPackages() == false)
2d11135a
AL
1034 continue;
1035
45415543 1036 if ((*I)->Exists() == false)
2d11135a 1037 continue;
b2e465d6 1038
45415543 1039 if ((*I)->FindInCache(Gen.GetCache()).end() == false)
a77ad7c3
AL
1040 {
1041 _error->Warning("Duplicate sources.list entry %s",
45415543 1042 (*I)->Describe().c_str());
a77ad7c3
AL
1043 continue;
1044 }
1045
45415543 1046 unsigned long Size = (*I)->Size();
2e5f4e45
DK
1047 if (Progress != NULL)
1048 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
b2e465d6 1049 CurrentSize += Size;
2d11135a 1050
45415543 1051 if ((*I)->Merge(Gen,Progress) == false)
b2e465d6
AL
1052 return false;
1053 }
45415543
AL
1054
1055 if (Gen.HasFileDeps() == true)
1056 {
2e5f4e45
DK
1057 if (Progress != NULL)
1058 Progress->Done();
45415543
AL
1059 TotalSize = ComputeSize(Start, End);
1060 CurrentSize = 0;
1061 for (I = Start; I != End; I++)
1062 {
1063 unsigned long Size = (*I)->Size();
2e5f4e45
DK
1064 if (Progress != NULL)
1065 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
45415543
AL
1066 CurrentSize += Size;
1067 if ((*I)->MergeFileProvides(Gen,Progress) == false)
1068 return false;
1069 }
1070 }
2d11135a 1071
b35d2f5f
AL
1072 return true;
1073}
1074 /*}}}*/
dd13742e 1075// CacheGenerator::CreateDynamicMMap - load an mmap with configuration options /*{{{*/
dcdf1ef1
DK
1076DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1077 unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1078 unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1079 unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1080 Flags |= MMap::Moveable;
1081 if (_config->FindB("APT::Cache-Fallback", false) == true)
1082 Flags |= MMap::Fallback;
1083 if (CacheF != NULL)
1084 return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1085 else
1086 return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1087}
dd13742e 1088 /*}}}*/
2e5f4e45 1089// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
b35d2f5f 1090// ---------------------------------------------------------------------
b2e465d6
AL
1091/* This makes sure that the status cache (the cache that has all
1092 index files from the sources list and all local ones) is ready
1093 to be mmaped. If OutMap is not zero then a MMap object representing
1094 the cache will be stored there. This is pretty much mandetory if you
1095 are using AllowMem. AllowMem lets the function be run as non-root
1096 where it builds the cache 'fast' into a memory buffer. */
2e5f4e45
DK
1097__deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1098 MMap **OutMap, bool AllowMem)
1099 { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1100bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
b2e465d6 1101 MMap **OutMap,bool AllowMem)
b35d2f5f 1102{
c8e572e3 1103 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
67db871e 1104
7db98ffc
MZ
1105 vector<pkgIndexFile *> Files;
1106 for (vector<metaIndex *>::const_iterator i = List.begin();
1107 i != List.end();
1108 i++)
1109 {
1110 vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1111 for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1112 j != Indexes->end();
1113 j++)
1114 Files.push_back (*j);
1115 }
1116
c8e572e3 1117 unsigned long const EndOfSource = Files.size();
b2e465d6
AL
1118 if (_system->AddStatusFiles(Files) == false)
1119 return false;
c5f44afc 1120
b2e465d6 1121 // Decide if we can write to the files..
c8e572e3
MV
1122 string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1123 string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1cd1c398
DK
1124
1125 // ensure the cache directory exists
1126 if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1127 {
1128 string dir = _config->FindDir("Dir::Cache");
1129 size_t const len = dir.size();
1130 if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1131 dir = dir.substr(0, len - 5);
1132 if (CacheFile.empty() == false)
1133 CreateDirectory(dir, flNotFile(CacheFile));
1134 if (SrcCacheFile.empty() == false)
1135 CreateDirectory(dir, flNotFile(SrcCacheFile));
1136 }
1137
b2e465d6
AL
1138 // Decide if we can write to the cache
1139 bool Writeable = false;
1140 if (CacheFile.empty() == false)
1141 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1142 else
1143 if (SrcCacheFile.empty() == false)
1144 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
c8e572e3
MV
1145 if (Debug == true)
1146 std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1147
b2e465d6
AL
1148 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1149 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
2e5f4e45
DK
1150
1151 if (Progress != NULL)
1152 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1153
b2e465d6
AL
1154 // Cache is OK, Fin.
1155 if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
1156 {
2e5f4e45
DK
1157 if (Progress != NULL)
1158 Progress->OverallProgress(1,1,1,_("Reading package lists"));
c8e572e3
MV
1159 if (Debug == true)
1160 std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
b2e465d6
AL
1161 return true;
1162 }
c8e572e3
MV
1163 else if (Debug == true)
1164 std::clog << "pkgcache.bin is NOT valid" << std::endl;
b2e465d6
AL
1165
1166 /* At this point we know we need to reconstruct the package cache,
1167 begin. */
1168 SPtr<FileFd> CacheF;
1169 SPtr<DynamicMMap> Map;
1170 if (Writeable == true && CacheFile.empty() == false)
1171 {
1172 unlink(CacheFile.c_str());
22041bd2 1173 CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
7a3c2ab0 1174 fchmod(CacheF->Fd(),0644);
dcdf1ef1 1175 Map = CreateDynamicMMap(CacheF, MMap::Public);
b35d2f5f
AL
1176 if (_error->PendingError() == true)
1177 return false;
c8e572e3
MV
1178 if (Debug == true)
1179 std::clog << "Open filebased MMap" << std::endl;
b35d2f5f 1180 }
b2e465d6 1181 else
8ce4327b 1182 {
b2e465d6 1183 // Just build it in memory..
dcdf1ef1 1184 Map = CreateDynamicMMap(NULL);
c8e572e3
MV
1185 if (Debug == true)
1186 std::clog << "Open memory Map (not filebased)" << std::endl;
8ce4327b 1187 }
b35d2f5f 1188
b2e465d6 1189 // Lets try the source cache.
b35d2f5f 1190 unsigned long CurrentSize = 0;
b2e465d6
AL
1191 unsigned long TotalSize = 0;
1192 if (CheckValidity(SrcCacheFile,Files.begin(),
1193 Files.begin()+EndOfSource) == true)
2d11135a 1194 {
c8e572e3
MV
1195 if (Debug == true)
1196 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
b2e465d6
AL
1197 // Preload the map with the source cache
1198 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
c8e572e3 1199 unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
eb162ff7
DK
1200 if ((alloc == 0 && _error->PendingError())
1201 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1202 SCacheF.Size()) == false)
b2e465d6
AL
1203 return false;
1204
1205 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
c8e572e3 1206
b2e465d6 1207 // Build the status cache
2e5f4e45 1208 pkgCacheGenerator Gen(Map.Get(),Progress);
2d11135a 1209 if (_error->PendingError() == true)
b2e465d6
AL
1210 return false;
1211 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1212 Files.begin()+EndOfSource,Files.end()) == false)
1213 return false;
25396fb0
DK
1214
1215 // FIXME: move me to a better place
1216 Gen.FinishCache(Progress);
b2e465d6
AL
1217 }
1218 else
2d11135a 1219 {
c8e572e3
MV
1220 if (Debug == true)
1221 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
b2e465d6 1222 TotalSize = ComputeSize(Files.begin(),Files.end());
2d11135a 1223
b2e465d6 1224 // Build the source cache
2e5f4e45 1225 pkgCacheGenerator Gen(Map.Get(),Progress);
b2e465d6
AL
1226 if (_error->PendingError() == true)
1227 return false;
1228 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1229 Files.begin(),Files.begin()+EndOfSource) == false)
1230 return false;
2d11135a 1231
b2e465d6
AL
1232 // Write it back
1233 if (Writeable == true && SrcCacheFile.empty() == false)
2d11135a 1234 {
22041bd2 1235 FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
b2e465d6
AL
1236 if (_error->PendingError() == true)
1237 return false;
7a3c2ab0
AL
1238
1239 fchmod(SCacheF.Fd(),0644);
1240
b2e465d6
AL
1241 // Write out the main data
1242 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1243 return _error->Error(_("IO Error saving source cache"));
1244 SCacheF.Sync();
1245
1246 // Write out the proper header
1247 Gen.GetCache().HeaderP->Dirty = false;
1248 if (SCacheF.Seek(0) == false ||
1249 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1250 return _error->Error(_("IO Error saving source cache"));
b2e465d6 1251 Gen.GetCache().HeaderP->Dirty = true;
7a3c2ab0 1252 SCacheF.Sync();
2d11135a
AL
1253 }
1254
b2e465d6
AL
1255 // Build the status cache
1256 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1257 Files.begin()+EndOfSource,Files.end()) == false)
1258 return false;
25396fb0
DK
1259
1260 // FIXME: move me to a better place
1261 Gen.FinishCache(Progress);
2d11135a 1262 }
c8e572e3
MV
1263 if (Debug == true)
1264 std::clog << "Caches are ready for shipping" << std::endl;
2d11135a 1265
b2e465d6
AL
1266 if (_error->PendingError() == true)
1267 return false;
1268 if (OutMap != 0)
2d11135a 1269 {
b2e465d6 1270 if (CacheF != 0)
2d11135a 1271 {
b2e465d6 1272 delete Map.UnGuard();
eb162ff7 1273 *OutMap = new MMap(*CacheF,0);
2d11135a 1274 }
b2e465d6
AL
1275 else
1276 {
1277 *OutMap = Map.UnGuard();
1278 }
2d11135a
AL
1279 }
1280
b2e465d6
AL
1281 return true;
1282}
1283 /*}}}*/
2e5f4e45 1284// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
b2e465d6
AL
1285// ---------------------------------------------------------------------
1286/* */
2e5f4e45
DK
1287__deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1288 { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1289bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
b2e465d6 1290{
b2e465d6
AL
1291 vector<pkgIndexFile *> Files;
1292 unsigned long EndOfSource = Files.size();
1293 if (_system->AddStatusFiles(Files) == false)
1294 return false;
dcdf1ef1
DK
1295
1296 SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
b2e465d6
AL
1297 unsigned long CurrentSize = 0;
1298 unsigned long TotalSize = 0;
1299
1300 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1301
1302 // Build the status cache
2e5f4e45
DK
1303 if (Progress != NULL)
1304 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1305 pkgCacheGenerator Gen(Map.Get(),Progress);
2d11135a 1306 if (_error->PendingError() == true)
b2e465d6
AL
1307 return false;
1308 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1309 Files.begin()+EndOfSource,Files.end()) == false)
1310 return false;
25396fb0
DK
1311
1312 // FIXME: move me to a better place
1313 Gen.FinishCache(Progress);
1314
b2e465d6
AL
1315 if (_error->PendingError() == true)
1316 return false;
1317 *OutMap = Map.UnGuard();
2d11135a 1318
b2e465d6 1319 return true;
2d11135a
AL
1320}
1321 /*}}}*/