]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
implement proper progress report with OpProgress
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
b2e465d6
AL
13#define APT_COMPATIBILITY 986
14
094a497d
AL
15#include <apt-pkg/pkgcachegen.h>
16#include <apt-pkg/error.h>
17#include <apt-pkg/version.h>
b35d2f5f
AL
18#include <apt-pkg/progress.h>
19#include <apt-pkg/sourcelist.h>
20#include <apt-pkg/configuration.h>
25396fb0 21#include <apt-pkg/aptconfiguration.h>
cdcc6d34 22#include <apt-pkg/strutl.h>
b2e465d6
AL
23#include <apt-pkg/sptr.h>
24#include <apt-pkg/pkgsystem.h>
aea7f4c8 25#include <apt-pkg/macros.h>
578bfd0a 26
afb1e2e3
MV
27#include <apt-pkg/tagfile.h>
28
b2e465d6 29#include <apti18n.h>
e7b470ee
AL
30
31#include <vector>
32
578bfd0a
AL
33#include <sys/stat.h>
34#include <unistd.h>
803fafcb 35#include <errno.h>
7ef72446 36#include <stdio.h>
578bfd0a 37 /*}}}*/
e7b470ee 38typedef vector<pkgIndexFile *>::iterator FileIterator;
d10cef82 39template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
578bfd0a
AL
40
41// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
42// ---------------------------------------------------------------------
25396fb0 43/* We set the dirty flag and make sure that is written to the disk */
b2e465d6 44pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45415543
AL
45 Map(*pMap), Cache(pMap,false), Progress(Prog),
46 FoundFileDeps(0)
578bfd0a 47{
ddc1d8d0 48 CurrentFile = 0;
b2e465d6 49 memset(UniqHash,0,sizeof(UniqHash));
ddc1d8d0 50
578bfd0a
AL
51 if (_error->PendingError() == true)
52 return;
b2e465d6 53
578bfd0a
AL
54 if (Map.Size() == 0)
55 {
b2e465d6
AL
56 // Setup the map interface..
57 Cache.HeaderP = (pkgCache::Header *)Map.Data();
c5f44afc
DK
58 if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
59 return;
60
b2e465d6 61 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
c5f44afc 62
b2e465d6 63 // Starting header
578bfd0a 64 *Cache.HeaderP = pkgCache::Header();
a9fe5928
DK
65 map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
66 Cache.HeaderP->VerSysName = idxVerSysName;
67 map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
68 Cache.HeaderP->Architecture = idxArchitecture;
69 if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
70 return;
c5f44afc 71 Cache.ReMap();
578bfd0a 72 }
b2e465d6
AL
73 else
74 {
75 // Map directly from the existing file
76 Cache.ReMap();
77 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
78 if (Cache.VS != _system->VS)
79 {
80 _error->Error(_("Cache has an incompatible versioning system"));
81 return;
82 }
83 }
84
578bfd0a
AL
85 Cache.HeaderP->Dirty = true;
86 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
87}
88 /*}}}*/
89// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
90// ---------------------------------------------------------------------
91/* We sync the data then unset the dirty flag in two steps so as to
92 advoid a problem during a crash */
93pkgCacheGenerator::~pkgCacheGenerator()
94{
95 if (_error->PendingError() == true)
96 return;
97 if (Map.Sync() == false)
98 return;
99
100 Cache.HeaderP->Dirty = false;
101 Map.Sync(0,sizeof(pkgCache::Header));
102}
103 /*}}}*/
a9fe5928
DK
104void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
105 if (oldMap == newMap)
106 return;
107
108 Cache.ReMap(false);
109
110 CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
111
112 for (size_t i = 0; i < _count(UniqHash); ++i)
113 if (UniqHash[i] != 0)
114 UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
115
7635093c 116 for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
a9fe5928 117 i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
f7a35f2e 118 (*i)->ReMap(oldMap, newMap);
7635093c 119 for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
a9fe5928 120 i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
f7a35f2e 121 (*i)->ReMap(oldMap, newMap);
7635093c 122 for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
a9fe5928 123 i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
f7a35f2e 124 (*i)->ReMap(oldMap, newMap);
7635093c 125 for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
a9fe5928 126 i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
f7a35f2e 127 (*i)->ReMap(oldMap, newMap);
7635093c 128 for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
a9fe5928 129 i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
f7a35f2e 130 (*i)->ReMap(oldMap, newMap);
7635093c 131 for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
a9fe5928 132 i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
f7a35f2e 133 (*i)->ReMap(oldMap, newMap);
7635093c 134 for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
a9fe5928 135 i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
f7a35f2e 136 (*i)->ReMap(oldMap, newMap);
a9fe5928 137} /*}}}*/
7e58ab0c 138// CacheGenerator::WriteStringInMap /*{{{*/
a9fe5928 139map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
7e58ab0c 140 const unsigned long &Len) {
a9fe5928
DK
141 void const * const oldMap = Map.Data();
142 map_ptrloc const index = Map.WriteString(String, Len);
143 if (index != 0)
144 ReMap(oldMap, Map.Data());
145 return index;
7e58ab0c
DK
146}
147 /*}}}*/
148// CacheGenerator::WriteStringInMap /*{{{*/
a9fe5928
DK
149map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
150 void const * const oldMap = Map.Data();
151 map_ptrloc const index = Map.WriteString(String);
152 if (index != 0)
153 ReMap(oldMap, Map.Data());
154 return index;
7e58ab0c
DK
155}
156 /*}}}*/
a9fe5928
DK
157map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
158 void const * const oldMap = Map.Data();
159 map_ptrloc const index = Map.Allocate(size);
160 if (index != 0)
161 ReMap(oldMap, Map.Data());
162 return index;
7e58ab0c
DK
163}
164 /*}}}*/
578bfd0a
AL
165// CacheGenerator::MergeList - Merge the package list /*{{{*/
166// ---------------------------------------------------------------------
167/* This provides the generation of the entries in the cache. Each loop
168 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
169bool pkgCacheGenerator::MergeList(ListParser &List,
170 pkgCache::VerIterator *OutVer)
578bfd0a
AL
171{
172 List.Owner = this;
0149949b 173
f9eec0e7 174 unsigned int Counter = 0;
0149949b 175 while (List.Step() == true)
578bfd0a 176 {
5bf15716 177 string const PackageName = List.Package();
65a1e968
AL
178 if (PackageName.empty() == true)
179 return false;
5bf15716 180
566046f4 181 string const Arch = List.Architecture();
28166356 182
8b32e920 183 // Get a pointer to the package structure
9ddf7030 184 pkgCache::PkgIterator Pkg;
a9fe5928 185 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
28166356 186 if (NewPackage(Pkg, PackageName, Arch) == false)
6804503b 187 return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
a246f2dc 188 Counter++;
ddc1d8d0
AL
189 if (Counter % 100 == 0 && Progress != 0)
190 Progress->Progress(List.Offset());
8ce4327b 191
578bfd0a
AL
192 /* Get a pointer to the version structure. We know the list is sorted
193 so we use that fact in the search. Insertion of new versions is
194 done with correct sorting */
195 string Version = List.Version();
f55a958f
AL
196 if (Version.empty() == true)
197 {
770c32ec
MV
198 // we first process the package, then the descriptions
199 // (this has the bonus that we get MMap error when we run out
200 // of MMap space)
32b9a14c 201 pkgCache::VerIterator Ver(Cache);
a9fe5928 202 Dynamic<pkgCache::VerIterator> DynVer(Ver);
32b9a14c 203 if (List.UsePackage(Pkg, Ver) == false)
6804503b 204 return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
7a3c2ab0 205 PackageName.c_str());
770c32ec 206
a52f938b
OS
207 // Find the right version to write the description
208 MD5SumValue CurMd5 = List.Description_md5();
32b9a14c 209 Ver = Pkg.VersionList();
770c32ec 210
a9fe5928 211 for (; Ver.end() == false; ++Ver)
a52f938b
OS
212 {
213 pkgCache::DescIterator Desc = Ver.DescriptionList();
a9fe5928 214 Dynamic<pkgCache::DescIterator> DynDesc(Desc);
a52f938b 215 map_ptrloc *LastDesc = &Ver->DescriptionList;
c76c44b1 216 bool duplicate=false;
217
218 // don't add a new description if we have one for the given
219 // md5 && language
9ee47c29 220 for ( ; Desc.end() == false; Desc++)
c76c44b1 221 if (MD5SumValue(Desc.md5()) == CurMd5 &&
222 Desc.LanguageCode() == List.DescriptionLanguage())
223 duplicate=true;
224 if(duplicate)
225 continue;
226
227 for (Desc = Ver.DescriptionList();
9ee47c29 228 Desc.end() == false;
c76c44b1 229 LastDesc = &Desc->NextDesc, Desc++)
770c32ec 230 {
770c32ec
MV
231 if (MD5SumValue(Desc.md5()) == CurMd5)
232 {
a52f938b 233 // Add new description
a9fe5928
DK
234 void const * const oldMap = Map.Data();
235 map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
236 if (oldMap != Map.Data())
237 LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
238 *LastDesc = descindex;
a52f938b 239 Desc->ParentPkg = Pkg.Index();
770c32ec 240
c5f44afc 241 if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
f0f66a3d 242 return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
a52f938b
OS
243 break;
244 }
770c32ec 245 }
a52f938b 246 }
770c32ec 247
f55a958f
AL
248 continue;
249 }
250
578bfd0a 251 pkgCache::VerIterator Ver = Pkg.VersionList();
a9fe5928 252 Dynamic<pkgCache::VerIterator> DynVer(Ver);
a52f938b 253 map_ptrloc *LastVer = &Pkg->VersionList;
a9fe5928 254 void const * oldMap = Map.Data();
2246928b 255 int Res = 1;
7e2b56a3 256 unsigned long const Hash = List.VersionHash();
9ee47c29 257 for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
578bfd0a 258 {
c24972cb 259 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
7e2b56a3
DK
260 // Version is higher as current version - insert here
261 if (Res > 0)
578bfd0a 262 break;
7e2b56a3
DK
263 // Versionstrings are equal - is hash also equal?
264 if (Res == 0 && Ver->Hash == Hash)
265 break;
266 // proceed with the next till we have either the right
267 // or we found another version (which will be lower)
578bfd0a 268 }
7e2b56a3
DK
269
270 /* We already have a version for this item, record that we saw it */
e426a5ff 271 if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
578bfd0a 272 {
f55a958f 273 if (List.UsePackage(Pkg,Ver) == false)
6804503b 274 return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
7a3c2ab0 275 PackageName.c_str());
f78439bf 276
578bfd0a 277 if (NewFileVer(Ver,List) == false)
6804503b 278 return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
7a3c2ab0 279 PackageName.c_str());
578bfd0a 280
ddc1d8d0
AL
281 // Read only a single record and return
282 if (OutVer != 0)
283 {
284 *OutVer = Ver;
45415543 285 FoundFileDeps |= List.HasFileDeps();
ddc1d8d0
AL
286 return true;
287 }
288
578bfd0a 289 continue;
204fbdcc
AL
290 }
291
578bfd0a 292 // Add a new version
a9fe5928
DK
293 map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
294 if (verindex == 0 && _error->PendingError())
295 return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
296 PackageName.c_str(), 1);
297
298 if (oldMap != Map.Data())
299 LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
300 *LastVer = verindex;
f55a958f 301 Ver->ParentPkg = Pkg.Index();
204fbdcc 302 Ver->Hash = Hash;
a52f938b 303
a9fe5928
DK
304 if (List.NewVersion(Ver) == false)
305 return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
306 PackageName.c_str(), 2);
0149949b 307
f55a958f 308 if (List.UsePackage(Pkg,Ver) == false)
6804503b 309 return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
7a3c2ab0 310 PackageName.c_str());
f55a958f 311
578bfd0a 312 if (NewFileVer(Ver,List) == false)
a9fe5928
DK
313 return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
314 PackageName.c_str(), 3);
ddc1d8d0
AL
315
316 // Read only a single record and return
317 if (OutVer != 0)
318 {
319 *OutVer = Ver;
45415543 320 FoundFileDeps |= List.HasFileDeps();
ddc1d8d0
AL
321 return true;
322 }
a52f938b
OS
323
324 /* Record the Description data. Description data always exist in
325 Packages and Translation-* files. */
326 pkgCache::DescIterator Desc = Ver.DescriptionList();
a9fe5928 327 Dynamic<pkgCache::DescIterator> DynDesc(Desc);
a52f938b 328 map_ptrloc *LastDesc = &Ver->DescriptionList;
a9fe5928 329
a52f938b 330 // Skip to the end of description set
9ee47c29 331 for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
a52f938b
OS
332
333 // Add new description
a9fe5928
DK
334 oldMap = Map.Data();
335 map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
336 if (oldMap != Map.Data())
337 LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
338 *LastDesc = descindex;
a52f938b
OS
339 Desc->ParentPkg = Pkg.Index();
340
c5f44afc 341 if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
f0f66a3d 342 return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
578bfd0a 343 }
0149949b 344
45415543
AL
345 FoundFileDeps |= List.HasFileDeps();
346
6a3da7a6
AL
347 if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
348 return _error->Error(_("Wow, you exceeded the number of package "
349 "names this APT is capable of."));
350 if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
351 return _error->Error(_("Wow, you exceeded the number of versions "
352 "this APT is capable of."));
a52f938b
OS
353 if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
354 return _error->Error(_("Wow, you exceeded the number of descriptions "
355 "this APT is capable of."));
6a3da7a6
AL
356 if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
357 return _error->Error(_("Wow, you exceeded the number of dependencies "
358 "this APT is capable of."));
578bfd0a
AL
359 return true;
360}
361 /*}}}*/
45415543
AL
362// CacheGenerator::MergeFileProvides - Merge file provides /*{{{*/
363// ---------------------------------------------------------------------
364/* If we found any file depends while parsing the main list we need to
365 resolve them. Since it is undesired to load the entire list of files
366 into the cache as virtual packages we do a two stage effort. MergeList
367 identifies the file depends and this creates Provdies for them by
368 re-parsing all the indexs. */
369bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
370{
371 List.Owner = this;
372
373 unsigned int Counter = 0;
374 while (List.Step() == true)
375 {
376 string PackageName = List.Package();
377 if (PackageName.empty() == true)
378 return false;
379 string Version = List.Version();
380 if (Version.empty() == true)
381 continue;
382
383 pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
a9fe5928 384 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
45415543 385 if (Pkg.end() == true)
6804503b 386 return _error->Error(_("Error occurred while processing %s (FindPkg)"),
45415543
AL
387 PackageName.c_str());
388 Counter++;
389 if (Counter % 100 == 0 && Progress != 0)
390 Progress->Progress(List.Offset());
391
392 unsigned long Hash = List.VersionHash();
393 pkgCache::VerIterator Ver = Pkg.VersionList();
a9fe5928 394 Dynamic<pkgCache::VerIterator> DynVer(Ver);
9ee47c29 395 for (; Ver.end() == false; Ver++)
45415543
AL
396 {
397 if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
398 {
399 if (List.CollectFileProvides(Cache,Ver) == false)
6804503b 400 return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
45415543
AL
401 break;
402 }
403 }
404
405 if (Ver.end() == true)
406 _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
407 }
408
409 return true;
410}
411 /*}}}*/
5bf15716
DK
412// CacheGenerator::NewGroup - Add a new group /*{{{*/
413// ---------------------------------------------------------------------
414/* This creates a new group structure and adds it to the hash table */
33dd02e3
DK
415bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
416{
417 Grp = Cache.FindGrp(Name);
418 if (Grp.end() == false)
419 return true;
5bf15716 420
33dd02e3 421 // Get a structure
a9fe5928 422 map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
33dd02e3
DK
423 if (unlikely(Group == 0))
424 return false;
5bf15716 425
33dd02e3 426 Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
a9fe5928
DK
427 map_ptrloc const idxName = WriteStringInMap(Name);
428 if (unlikely(idxName == 0))
33dd02e3 429 return false;
a9fe5928 430 Grp->Name = idxName;
5bf15716 431
33dd02e3
DK
432 // Insert it into the hash table
433 unsigned long const Hash = Cache.Hash(Name);
434 Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
435 Cache.HeaderP->GrpHashTable[Hash] = Group;
5bf15716 436
52c41485 437 Grp->ID = Cache.HeaderP->GroupCount++;
33dd02e3 438 return true;
5bf15716
DK
439}
440 /*}}}*/
578bfd0a
AL
441// CacheGenerator::NewPackage - Add a new package /*{{{*/
442// ---------------------------------------------------------------------
443/* This creates a new package structure and adds it to the hash table */
5bf15716
DK
444bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
445 const string &Arch) {
446 pkgCache::GrpIterator Grp;
a9fe5928 447 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
5bf15716
DK
448 if (unlikely(NewGroup(Grp, Name) == false))
449 return false;
450
451 Pkg = Grp.FindPkg(Arch);
452 if (Pkg.end() == false)
453 return true;
a52f938b 454
578bfd0a 455 // Get a structure
a9fe5928 456 map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
5bf15716 457 if (unlikely(Package == 0))
578bfd0a 458 return false;
f55a958f 459 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
5bf15716 460
c408e01e
DK
461 // Insert the package into our package list
462 if (Grp->FirstPackage == 0) // the group is new
463 {
464 // Insert it into the hash table
465 unsigned long const Hash = Cache.Hash(Name);
466 Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
467 Cache.HeaderP->PkgHashTable[Hash] = Package;
468 Grp->FirstPackage = Package;
469 }
470 else // Group the Packages together
471 {
472 // this package is the new last package
473 pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
474 Pkg->NextPackage = LastPkg->NextPackage;
475 LastPkg->NextPackage = Package;
476 }
477 Grp->LastPackage = Package;
5bf15716
DK
478
479 // Set the name, arch and the ID
480 Pkg->Name = Grp->Name;
481 Pkg->Group = Grp.Index();
566046f4 482 map_ptrloc const idxArch = WriteUniqString((Arch == "all") ? _config->Find("APT::Architecture") : Arch.c_str());
a9fe5928 483 if (unlikely(idxArch == 0))
578bfd0a 484 return false;
a9fe5928 485 Pkg->Arch = idxArch;
578bfd0a 486 Pkg->ID = Cache.HeaderP->PackageCount++;
5bf15716 487
578bfd0a
AL
488 return true;
489}
490 /*}}}*/
491// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
492// ---------------------------------------------------------------------
493/* */
f55a958f 494bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
495 ListParser &List)
496{
ddc1d8d0
AL
497 if (CurrentFile == 0)
498 return true;
499
dcb79bae 500 // Get a structure
a9fe5928 501 map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
dcb79bae
AL
502 if (VerFile == 0)
503 return 0;
504
505 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
506 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
507
508 // Link it to the end of the list
349cd3b8 509 map_ptrloc *Last = &Ver->FileList;
eb162ff7 510 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
03e39e59
AL
511 Last = &V->NextFile;
512 VF->NextFile = *Last;
513 *Last = VF.Index();
514
dcb79bae
AL
515 VF->Offset = List.Offset();
516 VF->Size = List.Size();
ad00ae81
AL
517 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
518 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
519 Cache.HeaderP->VerFileCount++;
520
f55a958f 521 return true;
578bfd0a
AL
522}
523 /*}}}*/
524// CacheGenerator::NewVersion - Create a new Version /*{{{*/
525// ---------------------------------------------------------------------
f55a958f 526/* This puts a version structure in the linked list */
578bfd0a 527unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
171c75f1 528 const string &VerStr,
578bfd0a
AL
529 unsigned long Next)
530{
f55a958f 531 // Get a structure
a9fe5928 532 map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
f55a958f 533 if (Version == 0)
0149949b 534 return 0;
f55a958f
AL
535
536 // Fill it in
537 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
f55a958f
AL
538 Ver->NextVer = Next;
539 Ver->ID = Cache.HeaderP->VersionCount++;
a9fe5928
DK
540 map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
541 if (unlikely(idxVerStr == 0))
0149949b 542 return 0;
a9fe5928 543 Ver->VerStr = idxVerStr;
f55a958f 544
0149949b 545 return Version;
578bfd0a
AL
546}
547 /*}}}*/
a52f938b
OS
548// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
549// ---------------------------------------------------------------------
550/* */
551bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
552 ListParser &List)
553{
554 if (CurrentFile == 0)
555 return true;
556
557 // Get a structure
a9fe5928 558 map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
a52f938b 559 if (DescFile == 0)
c5f44afc 560 return false;
770c32ec 561
a52f938b
OS
562 pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
563 DF->File = CurrentFile - Cache.PkgFileP;
770c32ec 564
a52f938b
OS
565 // Link it to the end of the list
566 map_ptrloc *Last = &Desc->FileList;
eb162ff7 567 for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
a52f938b 568 Last = &D->NextFile;
770c32ec 569
a52f938b
OS
570 DF->NextFile = *Last;
571 *Last = DF.Index();
572
573 DF->Offset = List.Offset();
574 DF->Size = List.Size();
575 if (Cache.HeaderP->MaxDescFileSize < DF->Size)
576 Cache.HeaderP->MaxDescFileSize = DF->Size;
577 Cache.HeaderP->DescFileCount++;
578
579 return true;
580}
581 /*}}}*/
582// CacheGenerator::NewDescription - Create a new Description /*{{{*/
583// ---------------------------------------------------------------------
584/* This puts a description structure in the linked list */
585map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
c76c44b1 586 const string &Lang,
587 const MD5SumValue &md5sum,
a52f938b
OS
588 map_ptrloc Next)
589{
590 // Get a structure
a9fe5928 591 map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
a52f938b
OS
592 if (Description == 0)
593 return 0;
594
595 // Fill it in
596 Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
597 Desc->NextDesc = Next;
598 Desc->ID = Cache.HeaderP->DescriptionCount++;
a9fe5928
DK
599 map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
600 map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
601 if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
c5f44afc 602 return 0;
a9fe5928
DK
603 Desc->language_code = idxlanguage_code;
604 Desc->md5sum = idxmd5sum;
a52f938b
OS
605
606 return Description;
607}
608 /*}}}*/
25396fb0
DK
609// CacheGenerator::FinishCache - do various finish operations /*{{{*/
610// ---------------------------------------------------------------------
611/* This prepares the Cache for delivery */
2e5f4e45 612bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
33dd02e3
DK
613{
614 // FIXME: add progress reporting for this operation
615 // Do we have different architectures in your groups ?
616 vector<string> archs = APT::Configuration::getArchitectures();
617 if (archs.size() > 1)
618 {
619 // Create Conflicts in between the group
a9fe5928
DK
620 pkgCache::GrpIterator G = GetCache().GrpBegin();
621 Dynamic<pkgCache::GrpIterator> DynG(G);
622 for (; G.end() != true; G++)
33dd02e3
DK
623 {
624 string const PkgName = G.Name();
a9fe5928
DK
625 pkgCache::PkgIterator P = G.PackageList();
626 Dynamic<pkgCache::PkgIterator> DynP(P);
627 for (; P.end() != true; P = G.NextPkg(P))
33dd02e3 628 {
33dd02e3 629 pkgCache::PkgIterator allPkg;
a9fe5928
DK
630 Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
631 pkgCache::VerIterator V = P.VersionList();
632 Dynamic<pkgCache::VerIterator> DynV(V);
633 for (; V.end() != true; V++)
33dd02e3 634 {
a1ac2ca8 635 char const * const Arch = P.Arch();
33dd02e3
DK
636 map_ptrloc *OldDepLast = NULL;
637 /* MultiArch handling introduces a lot of implicit Dependencies:
638 - MultiArch: same → Co-Installable if they have the same version
639 - Architecture: all → Need to be Co-Installable for internal reasons
640 - All others conflict with all other group members */
ca238ede 641 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
33dd02e3
DK
642 for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
643 {
6d38011b 644 if (Arch == 0 || *A == Arch)
33dd02e3
DK
645 continue;
646 /* We allow only one installed arch at the time
647 per group, therefore each group member conflicts
648 with all other group members */
649 pkgCache::PkgIterator D = G.FindPkg(*A);
a9fe5928 650 Dynamic<pkgCache::PkgIterator> DynD(D);
33dd02e3
DK
651 if (D.end() == true)
652 continue;
653 if (coInstall == true)
654 {
655 // Replaces: ${self}:other ( << ${binary:Version})
656 NewDepends(D, V, V.VerStr(),
657 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
658 OldDepLast);
659 // Breaks: ${self}:other (!= ${binary:Version})
660 NewDepends(D, V, V.VerStr(),
3b527295 661 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
33dd02e3 662 OldDepLast);
33dd02e3
DK
663 } else {
664 // Conflicts: ${self}:other
28166356
DK
665 NewDepends(D, V, "",
666 pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
667 OldDepLast);
33dd02e3
DK
668 }
669 }
670 }
671 }
672 }
673 }
674 return true;
25396fb0
DK
675}
676 /*}}}*/
677// CacheGenerator::NewDepends - Create a dependency element /*{{{*/
dcb79bae
AL
678// ---------------------------------------------------------------------
679/* This creates a dependency element in the tree. It is linked to the
680 version and to the package that it is pointing to. */
25396fb0
DK
681bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
682 pkgCache::VerIterator &Ver,
683 string const &Version,
684 unsigned int const &Op,
685 unsigned int const &Type,
686 map_ptrloc *OldDepLast)
dcb79bae 687{
a9fe5928 688 void const * const oldMap = Map.Data();
dcb79bae 689 // Get a structure
a9fe5928 690 map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
5bf15716 691 if (unlikely(Dependency == 0))
dcb79bae
AL
692 return false;
693
694 // Fill it in
695 pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
a9fe5928 696 Dynamic<pkgCache::DepIterator> DynDep(Dep);
dcb79bae
AL
697 Dep->ParentVer = Ver.Index();
698 Dep->Type = Type;
699 Dep->CompareOp = Op;
700 Dep->ID = Cache.HeaderP->DependsCount++;
5bf15716 701
dcb79bae
AL
702 // Probe the reverse dependency list for a version string that matches
703 if (Version.empty() == false)
704 {
b2e465d6 705/* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
dcb79bae 706 if (I->Version != 0 && I.TargetVer() == Version)
f9eec0e7 707 Dep->Version = I->Version;*/
a9fe5928
DK
708 if (Dep->Version == 0) {
709 map_ptrloc const index = WriteStringInMap(Version);
710 if (unlikely(index == 0))
dcb79bae 711 return false;
a9fe5928
DK
712 Dep->Version = index;
713 }
dcb79bae 714 }
25396fb0 715
dcb79bae
AL
716 // Link it to the package
717 Dep->Package = Pkg.Index();
718 Dep->NextRevDepends = Pkg->RevDepends;
719 Pkg->RevDepends = Dep.Index();
25396fb0
DK
720
721 // Do we know where to link the Dependency to?
722 if (OldDepLast == NULL)
c1a22377 723 {
f9eec0e7 724 OldDepLast = &Ver->DependsList;
eb162ff7 725 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
f9eec0e7 726 OldDepLast = &D->NextDepends;
a9fe5928
DK
727 } else if (oldMap != Map.Data())
728 OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
45415543 729
f9eec0e7
AL
730 Dep->NextDepends = *OldDepLast;
731 *OldDepLast = Dep.Index();
732 OldDepLast = &Dep->NextDepends;
c1a22377 733
dcb79bae
AL
734 return true;
735}
736 /*}}}*/
25396fb0
DK
737// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
738// ---------------------------------------------------------------------
739/* This creates a Group and the Package to link this dependency to if
740 needed and handles also the caching of the old endpoint */
32b9a14c 741bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
25396fb0
DK
742 const string &PackageName,
743 const string &Arch,
744 const string &Version,
745 unsigned int Op,
746 unsigned int Type)
747{
748 pkgCache::GrpIterator Grp;
a9fe5928 749 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
25396fb0
DK
750 if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
751 return false;
752
753 // Locate the target package
754 pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
a9fe5928 755 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
25396fb0
DK
756 if (Pkg.end() == true) {
757 if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
758 return false;
759 }
760
761 // Is it a file dependency?
762 if (unlikely(PackageName[0] == '/'))
763 FoundFileDeps = true;
764
765 /* Caching the old end point speeds up generation substantially */
766 if (OldDepVer != Ver) {
767 OldDepLast = NULL;
768 OldDepVer = Ver;
769 }
770
771 return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
772}
773 /*}}}*/
dcb79bae
AL
774// ListParser::NewProvides - Create a Provides element /*{{{*/
775// ---------------------------------------------------------------------
776/* */
32b9a14c 777bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
67e0766f
DK
778 const string &PkgName,
779 const string &PkgArch,
171c75f1 780 const string &Version)
dcb79bae
AL
781{
782 pkgCache &Cache = Owner->Cache;
8efa2a3b
AL
783
784 // We do not add self referencing provides
566046f4
DK
785 if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
786 (PkgArch == "all" && _config->Find("APT::Architecture") == Ver.ParentPkg().Arch())))
8efa2a3b 787 return true;
dcb79bae
AL
788
789 // Get a structure
a9fe5928 790 map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
5bf15716 791 if (unlikely(Provides == 0))
dcb79bae 792 return false;
a7e66b17 793 Cache.HeaderP->ProvidesCount++;
dcb79bae
AL
794
795 // Fill it in
796 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
a9fe5928 797 Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
dcb79bae
AL
798 Prv->Version = Ver.Index();
799 Prv->NextPkgProv = Ver->ProvidesList;
800 Ver->ProvidesList = Prv.Index();
5bf15716 801 if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
dcb79bae
AL
802 return false;
803
804 // Locate the target package
8efa2a3b 805 pkgCache::PkgIterator Pkg;
a9fe5928 806 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
67e0766f 807 if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
8efa2a3b 808 return false;
dcb79bae
AL
809
810 // Link it to the package
811 Prv->ParentPkg = Pkg.Index();
812 Prv->NextProvides = Pkg->ProvidesList;
813 Pkg->ProvidesList = Prv.Index();
814
815 return true;
816}
817 /*}}}*/
578bfd0a
AL
818// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
819// ---------------------------------------------------------------------
820/* This is used to select which file is to be associated with all newly
b2e465d6 821 added versions. The caller is responsible for setting the IMS fields. */
171c75f1 822bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
b2e465d6
AL
823 const pkgIndexFile &Index,
824 unsigned long Flags)
578bfd0a 825{
578bfd0a 826 // Get some space for the structure
a9fe5928
DK
827 map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
828 if (unlikely(idxFile == 0))
578bfd0a 829 return false;
a9fe5928
DK
830 CurrentFile = Cache.PkgFileP + idxFile;
831
578bfd0a 832 // Fill it in
a9fe5928
DK
833 map_ptrloc const idxFileName = WriteStringInMap(File);
834 map_ptrloc const idxSite = WriteUniqString(Site);
835 if (unlikely(idxFileName == 0 || idxSite == 0))
836 return false;
837 CurrentFile->FileName = idxFileName;
838 CurrentFile->Site = idxSite;
578bfd0a
AL
839 CurrentFile->NextFile = Cache.HeaderP->FileList;
840 CurrentFile->Flags = Flags;
e1b74f61 841 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
a9fe5928
DK
842 map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
843 if (unlikely(idxIndexType == 0))
844 return false;
845 CurrentFile->IndexType = idxIndexType;
578bfd0a 846 PkgFileName = File;
ad00ae81 847 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 848 Cache.HeaderP->PackageFileCount++;
b2e465d6 849
ddc1d8d0 850 if (Progress != 0)
b2e465d6 851 Progress->SubProgress(Index.Size());
8efa2a3b 852 return true;
578bfd0a
AL
853}
854 /*}}}*/
f55a958f
AL
855// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
856// ---------------------------------------------------------------------
857/* This is used to create handles to strings. Given the same text it
858 always returns the same number */
859unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
860 unsigned int Size)
861{
f9eec0e7
AL
862 /* We use a very small transient hash table here, this speeds up generation
863 by a fair amount on slower machines */
864 pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
865 if (Bucket != 0 &&
866 stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
867 return Bucket->String;
868
f55a958f
AL
869 // Search for an insertion point
870 pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
871 int Res = 1;
349cd3b8 872 map_ptrloc *Last = &Cache.HeaderP->StringList;
f55a958f
AL
873 for (; I != Cache.StringItemP; Last = &I->NextItem,
874 I = Cache.StringItemP + I->NextItem)
875 {
9c14e3d6 876 Res = stringcmp(S,S+Size,Cache.StrP + I->String);
f55a958f
AL
877 if (Res >= 0)
878 break;
879 }
880
881 // Match
882 if (Res == 0)
f9eec0e7
AL
883 {
884 Bucket = I;
0149949b 885 return I->String;
f9eec0e7 886 }
f55a958f
AL
887
888 // Get a structure
a9fe5928
DK
889 void const * const oldMap = Map.Data();
890 map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
f55a958f 891 if (Item == 0)
0149949b
AL
892 return 0;
893
a9fe5928
DK
894 map_ptrloc const idxString = WriteStringInMap(S,Size);
895 if (unlikely(idxString == 0))
896 return 0;
897 if (oldMap != Map.Data()) {
898 Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
899 I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
900 }
901 *Last = Item;
902
f55a958f
AL
903 // Fill in the structure
904 pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
905 ItemP->NextItem = I - Cache.StringItemP;
a9fe5928
DK
906 ItemP->String = idxString;
907
f9eec0e7 908 Bucket = ItemP;
0149949b 909 return ItemP->String;
f55a958f
AL
910}
911 /*}}}*/
b2e465d6 912// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 913// ---------------------------------------------------------------------
b2e465d6
AL
914/* This just verifies that each file in the list of index files exists,
915 has matching attributes with the cache and the cache does not have
916 any extra files. */
171c75f1 917static bool CheckValidity(const string &CacheFile, FileIterator Start,
e7b470ee 918 FileIterator End,MMap **OutMap = 0)
b35d2f5f 919{
c8e572e3 920 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
b2e465d6
AL
921 // No file, certainly invalid
922 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
c8e572e3
MV
923 {
924 if (Debug == true)
925 std::clog << "CacheFile doesn't exist" << std::endl;
b35d2f5f 926 return false;
c8e572e3
MV
927 }
928
b2e465d6 929 // Map it
b35d2f5f 930 FileFd CacheF(CacheFile,FileFd::ReadOnly);
eb162ff7 931 SPtr<MMap> Map = new MMap(CacheF,0);
b35d2f5f 932 pkgCache Cache(Map);
b2e465d6 933 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f 934 {
c8e572e3
MV
935 if (Debug == true)
936 std::clog << "Errors are pending or Map is empty()" << std::endl;
b35d2f5f
AL
937 _error->Discard();
938 return false;
939 }
b35d2f5f 940
b2e465d6
AL
941 /* Now we check every index file, see if it is in the cache,
942 verify the IMS data and check that it is on the disk too.. */
943 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
944 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
945 for (; Start != End; Start++)
c8e572e3
MV
946 {
947 if (Debug == true)
948 std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
b2e465d6 949 if ((*Start)->HasPackages() == false)
c8e572e3
MV
950 {
951 if (Debug == true)
952 std::clog << "Has NO packages" << std::endl;
b2e465d6 953 continue;
c8e572e3 954 }
a77ad7c3 955
b2e465d6 956 if ((*Start)->Exists() == false)
b35d2f5f 957 {
a791a450 958#if 0 // mvo: we no longer give a message here (Default Sources spec)
b2e465d6
AL
959 _error->WarningE("stat",_("Couldn't stat source package list %s"),
960 (*Start)->Describe().c_str());
a791a450 961#endif
c8e572e3
MV
962 if (Debug == true)
963 std::clog << "file doesn't exist" << std::endl;
b2e465d6 964 continue;
b35d2f5f 965 }
b2e465d6
AL
966
967 // FindInCache is also expected to do an IMS check.
968 pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
969 if (File.end() == true)
c8e572e3
MV
970 {
971 if (Debug == true)
972 std::clog << "FindInCache returned end-Pointer" << std::endl;
b35d2f5f 973 return false;
c8e572e3 974 }
a52f938b 975
b2e465d6 976 Visited[File->ID] = true;
c8e572e3
MV
977 if (Debug == true)
978 std::clog << "with ID " << File->ID << " is valid" << std::endl;
b35d2f5f
AL
979 }
980
b2e465d6
AL
981 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
982 if (Visited[I] == false)
c8e572e3
MV
983 {
984 if (Debug == true)
985 std::clog << "File with ID" << I << " wasn't visited" << std::endl;
b2e465d6 986 return false;
c8e572e3 987 }
b35d2f5f 988
b35d2f5f
AL
989 if (_error->PendingError() == true)
990 {
c8e572e3
MV
991 if (Debug == true)
992 {
993 std::clog << "Validity failed because of pending errors:" << std::endl;
994 _error->DumpErrors();
995 }
b35d2f5f
AL
996 _error->Discard();
997 return false;
998 }
b35d2f5f 999
b2e465d6
AL
1000 if (OutMap != 0)
1001 *OutMap = Map.UnGuard();
b35d2f5f
AL
1002 return true;
1003}
1004 /*}}}*/
b2e465d6 1005// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 1006// ---------------------------------------------------------------------
b2e465d6
AL
1007/* Size is kind of an abstract notion that is only used for the progress
1008 meter */
e7b470ee 1009static unsigned long ComputeSize(FileIterator Start,FileIterator End)
b35d2f5f 1010{
b2e465d6
AL
1011 unsigned long TotalSize = 0;
1012 for (; Start != End; Start++)
b35d2f5f 1013 {
b2e465d6
AL
1014 if ((*Start)->HasPackages() == false)
1015 continue;
1016 TotalSize += (*Start)->Size();
b35d2f5f 1017 }
b2e465d6 1018 return TotalSize;
2d11135a
AL
1019}
1020 /*}}}*/
b2e465d6 1021// BuildCache - Merge the list of index files into the cache /*{{{*/
2d11135a 1022// ---------------------------------------------------------------------
b2e465d6
AL
1023/* */
1024static bool BuildCache(pkgCacheGenerator &Gen,
2e5f4e45 1025 OpProgress *Progress,
b2e465d6 1026 unsigned long &CurrentSize,unsigned long TotalSize,
e7b470ee 1027 FileIterator Start, FileIterator End)
2d11135a 1028{
45415543
AL
1029 FileIterator I;
1030 for (I = Start; I != End; I++)
2d11135a 1031 {
45415543 1032 if ((*I)->HasPackages() == false)
2d11135a
AL
1033 continue;
1034
45415543 1035 if ((*I)->Exists() == false)
2d11135a 1036 continue;
b2e465d6 1037
45415543 1038 if ((*I)->FindInCache(Gen.GetCache()).end() == false)
a77ad7c3
AL
1039 {
1040 _error->Warning("Duplicate sources.list entry %s",
45415543 1041 (*I)->Describe().c_str());
a77ad7c3
AL
1042 continue;
1043 }
1044
45415543 1045 unsigned long Size = (*I)->Size();
2e5f4e45
DK
1046 if (Progress != NULL)
1047 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
b2e465d6 1048 CurrentSize += Size;
2d11135a 1049
45415543 1050 if ((*I)->Merge(Gen,Progress) == false)
b2e465d6
AL
1051 return false;
1052 }
45415543
AL
1053
1054 if (Gen.HasFileDeps() == true)
1055 {
2e5f4e45
DK
1056 if (Progress != NULL)
1057 Progress->Done();
45415543
AL
1058 TotalSize = ComputeSize(Start, End);
1059 CurrentSize = 0;
1060 for (I = Start; I != End; I++)
1061 {
1062 unsigned long Size = (*I)->Size();
2e5f4e45
DK
1063 if (Progress != NULL)
1064 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
45415543
AL
1065 CurrentSize += Size;
1066 if ((*I)->MergeFileProvides(Gen,Progress) == false)
1067 return false;
1068 }
1069 }
2d11135a 1070
b35d2f5f
AL
1071 return true;
1072}
1073 /*}}}*/
dd13742e 1074// CacheGenerator::CreateDynamicMMap - load an mmap with configuration options /*{{{*/
dcdf1ef1
DK
1075DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1076 unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1077 unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1078 unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1079 Flags |= MMap::Moveable;
1080 if (_config->FindB("APT::Cache-Fallback", false) == true)
1081 Flags |= MMap::Fallback;
1082 if (CacheF != NULL)
1083 return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1084 else
1085 return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1086}
dd13742e 1087 /*}}}*/
2e5f4e45 1088// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
b35d2f5f 1089// ---------------------------------------------------------------------
b2e465d6
AL
1090/* This makes sure that the status cache (the cache that has all
1091 index files from the sources list and all local ones) is ready
1092 to be mmaped. If OutMap is not zero then a MMap object representing
1093 the cache will be stored there. This is pretty much mandetory if you
1094 are using AllowMem. AllowMem lets the function be run as non-root
1095 where it builds the cache 'fast' into a memory buffer. */
2e5f4e45
DK
1096__deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1097 MMap **OutMap, bool AllowMem)
1098 { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1099bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
b2e465d6 1100 MMap **OutMap,bool AllowMem)
b35d2f5f 1101{
c8e572e3 1102 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
67db871e 1103
7db98ffc
MZ
1104 vector<pkgIndexFile *> Files;
1105 for (vector<metaIndex *>::const_iterator i = List.begin();
1106 i != List.end();
1107 i++)
1108 {
1109 vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1110 for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1111 j != Indexes->end();
1112 j++)
1113 Files.push_back (*j);
1114 }
1115
c8e572e3 1116 unsigned long const EndOfSource = Files.size();
b2e465d6
AL
1117 if (_system->AddStatusFiles(Files) == false)
1118 return false;
c5f44afc 1119
b2e465d6 1120 // Decide if we can write to the files..
c8e572e3
MV
1121 string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1122 string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1cd1c398
DK
1123
1124 // ensure the cache directory exists
1125 if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1126 {
1127 string dir = _config->FindDir("Dir::Cache");
1128 size_t const len = dir.size();
1129 if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1130 dir = dir.substr(0, len - 5);
1131 if (CacheFile.empty() == false)
1132 CreateDirectory(dir, flNotFile(CacheFile));
1133 if (SrcCacheFile.empty() == false)
1134 CreateDirectory(dir, flNotFile(SrcCacheFile));
1135 }
1136
b2e465d6
AL
1137 // Decide if we can write to the cache
1138 bool Writeable = false;
1139 if (CacheFile.empty() == false)
1140 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1141 else
1142 if (SrcCacheFile.empty() == false)
1143 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
c8e572e3
MV
1144 if (Debug == true)
1145 std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1146
b2e465d6
AL
1147 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1148 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
2e5f4e45
DK
1149
1150 if (Progress != NULL)
1151 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1152
b2e465d6
AL
1153 // Cache is OK, Fin.
1154 if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
1155 {
2e5f4e45
DK
1156 if (Progress != NULL)
1157 Progress->OverallProgress(1,1,1,_("Reading package lists"));
c8e572e3
MV
1158 if (Debug == true)
1159 std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
b2e465d6
AL
1160 return true;
1161 }
c8e572e3
MV
1162 else if (Debug == true)
1163 std::clog << "pkgcache.bin is NOT valid" << std::endl;
b2e465d6
AL
1164
1165 /* At this point we know we need to reconstruct the package cache,
1166 begin. */
1167 SPtr<FileFd> CacheF;
1168 SPtr<DynamicMMap> Map;
1169 if (Writeable == true && CacheFile.empty() == false)
1170 {
1171 unlink(CacheFile.c_str());
22041bd2 1172 CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
7a3c2ab0 1173 fchmod(CacheF->Fd(),0644);
dcdf1ef1 1174 Map = CreateDynamicMMap(CacheF, MMap::Public);
b35d2f5f
AL
1175 if (_error->PendingError() == true)
1176 return false;
c8e572e3
MV
1177 if (Debug == true)
1178 std::clog << "Open filebased MMap" << std::endl;
b35d2f5f 1179 }
b2e465d6 1180 else
8ce4327b 1181 {
b2e465d6 1182 // Just build it in memory..
dcdf1ef1 1183 Map = CreateDynamicMMap(NULL);
c8e572e3
MV
1184 if (Debug == true)
1185 std::clog << "Open memory Map (not filebased)" << std::endl;
8ce4327b 1186 }
b35d2f5f 1187
b2e465d6 1188 // Lets try the source cache.
b35d2f5f 1189 unsigned long CurrentSize = 0;
b2e465d6
AL
1190 unsigned long TotalSize = 0;
1191 if (CheckValidity(SrcCacheFile,Files.begin(),
1192 Files.begin()+EndOfSource) == true)
2d11135a 1193 {
c8e572e3
MV
1194 if (Debug == true)
1195 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
b2e465d6
AL
1196 // Preload the map with the source cache
1197 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
c8e572e3 1198 unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
eb162ff7
DK
1199 if ((alloc == 0 && _error->PendingError())
1200 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1201 SCacheF.Size()) == false)
b2e465d6
AL
1202 return false;
1203
1204 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
c8e572e3 1205
b2e465d6 1206 // Build the status cache
2e5f4e45 1207 pkgCacheGenerator Gen(Map.Get(),Progress);
2d11135a 1208 if (_error->PendingError() == true)
b2e465d6
AL
1209 return false;
1210 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1211 Files.begin()+EndOfSource,Files.end()) == false)
1212 return false;
25396fb0
DK
1213
1214 // FIXME: move me to a better place
1215 Gen.FinishCache(Progress);
b2e465d6
AL
1216 }
1217 else
2d11135a 1218 {
c8e572e3
MV
1219 if (Debug == true)
1220 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
b2e465d6 1221 TotalSize = ComputeSize(Files.begin(),Files.end());
2d11135a 1222
b2e465d6 1223 // Build the source cache
2e5f4e45 1224 pkgCacheGenerator Gen(Map.Get(),Progress);
b2e465d6
AL
1225 if (_error->PendingError() == true)
1226 return false;
1227 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1228 Files.begin(),Files.begin()+EndOfSource) == false)
1229 return false;
2d11135a 1230
b2e465d6
AL
1231 // Write it back
1232 if (Writeable == true && SrcCacheFile.empty() == false)
2d11135a 1233 {
22041bd2 1234 FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
b2e465d6
AL
1235 if (_error->PendingError() == true)
1236 return false;
7a3c2ab0
AL
1237
1238 fchmod(SCacheF.Fd(),0644);
1239
b2e465d6
AL
1240 // Write out the main data
1241 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1242 return _error->Error(_("IO Error saving source cache"));
1243 SCacheF.Sync();
1244
1245 // Write out the proper header
1246 Gen.GetCache().HeaderP->Dirty = false;
1247 if (SCacheF.Seek(0) == false ||
1248 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1249 return _error->Error(_("IO Error saving source cache"));
b2e465d6 1250 Gen.GetCache().HeaderP->Dirty = true;
7a3c2ab0 1251 SCacheF.Sync();
2d11135a
AL
1252 }
1253
b2e465d6
AL
1254 // Build the status cache
1255 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1256 Files.begin()+EndOfSource,Files.end()) == false)
1257 return false;
25396fb0
DK
1258
1259 // FIXME: move me to a better place
1260 Gen.FinishCache(Progress);
2d11135a 1261 }
c8e572e3
MV
1262 if (Debug == true)
1263 std::clog << "Caches are ready for shipping" << std::endl;
2d11135a 1264
b2e465d6
AL
1265 if (_error->PendingError() == true)
1266 return false;
1267 if (OutMap != 0)
2d11135a 1268 {
b2e465d6 1269 if (CacheF != 0)
2d11135a 1270 {
b2e465d6 1271 delete Map.UnGuard();
eb162ff7 1272 *OutMap = new MMap(*CacheF,0);
2d11135a 1273 }
b2e465d6
AL
1274 else
1275 {
1276 *OutMap = Map.UnGuard();
1277 }
2d11135a
AL
1278 }
1279
b2e465d6
AL
1280 return true;
1281}
1282 /*}}}*/
2e5f4e45 1283// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
b2e465d6
AL
1284// ---------------------------------------------------------------------
1285/* */
2e5f4e45
DK
1286__deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1287 { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1288bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
b2e465d6 1289{
b2e465d6
AL
1290 vector<pkgIndexFile *> Files;
1291 unsigned long EndOfSource = Files.size();
1292 if (_system->AddStatusFiles(Files) == false)
1293 return false;
dcdf1ef1
DK
1294
1295 SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
b2e465d6
AL
1296 unsigned long CurrentSize = 0;
1297 unsigned long TotalSize = 0;
1298
1299 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1300
1301 // Build the status cache
2e5f4e45
DK
1302 if (Progress != NULL)
1303 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1304 pkgCacheGenerator Gen(Map.Get(),Progress);
2d11135a 1305 if (_error->PendingError() == true)
b2e465d6
AL
1306 return false;
1307 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1308 Files.begin()+EndOfSource,Files.end()) == false)
1309 return false;
25396fb0
DK
1310
1311 // FIXME: move me to a better place
1312 Gen.FinishCache(Progress);
1313
b2e465d6
AL
1314 if (_error->PendingError() == true)
1315 return false;
1316 *OutMap = Map.UnGuard();
2d11135a 1317
b2e465d6 1318 return true;
2d11135a
AL
1319}
1320 /*}}}*/