]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
add volatile sources support in libapt-pkg
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140 13#include <config.h>
b2e465d6 14
094a497d
AL
15#include <apt-pkg/pkgcachegen.h>
16#include <apt-pkg/error.h>
17#include <apt-pkg/version.h>
b35d2f5f
AL
18#include <apt-pkg/progress.h>
19#include <apt-pkg/sourcelist.h>
20#include <apt-pkg/configuration.h>
cdcc6d34 21#include <apt-pkg/strutl.h>
b2e465d6
AL
22#include <apt-pkg/sptr.h>
23#include <apt-pkg/pkgsystem.h>
aea7f4c8 24#include <apt-pkg/macros.h>
472ff00e
DK
25#include <apt-pkg/metaindex.h>
26#include <apt-pkg/fileutl.h>
453b82a3
DK
27#include <apt-pkg/hashsum_template.h>
28#include <apt-pkg/indexfile.h>
29#include <apt-pkg/md5.h>
30#include <apt-pkg/mmap.h>
31#include <apt-pkg/pkgcache.h>
32#include <apt-pkg/cacheiterators.h>
33
34#include <stddef.h>
35#include <string.h>
36#include <iostream>
37#include <string>
e7b470ee 38#include <vector>
5465192b
DK
39#include <memory>
40#include <algorithm>
578bfd0a
AL
41#include <sys/stat.h>
42#include <unistd.h>
ea542140
DK
43
44#include <apti18n.h>
578bfd0a 45 /*}}}*/
73688d27 46typedef std::vector<pkgIndexFile *>::iterator FileIterator;
d10cef82 47template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
578bfd0a 48
60683765 49static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
5f4db009
DK
50 MD5SumValue const &CurMd5, std::string const &CurLang);
51
73688d27
DK
52using std::string;
53
578bfd0a
AL
54// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
55// ---------------------------------------------------------------------
25396fb0 56/* We set the dirty flag and make sure that is written to the disk */
b2e465d6 57pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45415543 58 Map(*pMap), Cache(pMap,false), Progress(Prog),
6c55f07a 59 CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0), d(NULL)
578bfd0a
AL
60{
61 if (_error->PendingError() == true)
62 return;
b2e465d6 63
578bfd0a
AL
64 if (Map.Size() == 0)
65 {
b2e465d6
AL
66 // Setup the map interface..
67 Cache.HeaderP = (pkgCache::Header *)Map.Data();
c5f44afc
DK
68 if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
69 return;
70
b2e465d6 71 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
c5f44afc 72
b2e465d6 73 // Starting header
578bfd0a 74 *Cache.HeaderP = pkgCache::Header();
e8a7b0b2
DK
75
76 // make room for the hashtables for packages and groups
32ab4bd0 77 if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
e8a7b0b2
DK
78 return;
79
4ad8619b 80 map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
7a223b93
DK
81 if (unlikely(idxVerSysName == 0))
82 return;
a9fe5928 83 Cache.HeaderP->VerSysName = idxVerSysName;
78a5476f 84 map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
7a223b93 85 if (unlikely(idxArchitecture == 0))
a9fe5928 86 return;
7a223b93
DK
87 Cache.HeaderP->Architecture = idxArchitecture;
88
89 std::vector<std::string> archs = APT::Configuration::getArchitectures();
90 if (archs.size() > 1)
91 {
92 std::vector<std::string>::const_iterator a = archs.begin();
93 std::string list = *a;
94 for (++a; a != archs.end(); ++a)
95 list.append(",").append(*a);
4ad8619b 96 map_stringitem_t const idxArchitectures = WriteStringInMap(list);
7a223b93
DK
97 if (unlikely(idxArchitectures == 0))
98 return;
32ab4bd0 99 Cache.HeaderP->SetArchitectures(idxArchitectures);
7a223b93
DK
100 }
101 else
32ab4bd0 102 Cache.HeaderP->SetArchitectures(idxArchitecture);
7a223b93 103
c5f44afc 104 Cache.ReMap();
578bfd0a 105 }
b2e465d6
AL
106 else
107 {
108 // Map directly from the existing file
109 Cache.ReMap();
110 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
111 if (Cache.VS != _system->VS)
112 {
113 _error->Error(_("Cache has an incompatible versioning system"));
114 return;
e8a7b0b2 115 }
b2e465d6 116 }
e8a7b0b2 117
578bfd0a
AL
118 Cache.HeaderP->Dirty = true;
119 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
120}
121 /*}}}*/
122// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
123// ---------------------------------------------------------------------
124/* We sync the data then unset the dirty flag in two steps so as to
125 advoid a problem during a crash */
126pkgCacheGenerator::~pkgCacheGenerator()
127{
128 if (_error->PendingError() == true)
129 return;
130 if (Map.Sync() == false)
131 return;
132
133 Cache.HeaderP->Dirty = false;
1dfda2ce 134 Cache.HeaderP->CacheFileSize = Map.Size();
578bfd0a
AL
135 Map.Sync(0,sizeof(pkgCache::Header));
136}
137 /*}}}*/
a9fe5928
DK
138void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
139 if (oldMap == newMap)
140 return;
141
64dda04b
MV
142 if (_config->FindB("Debug::pkgCacheGen", false))
143 std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
144
a9fe5928
DK
145 Cache.ReMap(false);
146
cf4ff3b7 147 CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
b07aeb1a 148 CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
a9fe5928 149
7635093c 150 for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
a9fe5928 151 i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
f7a35f2e 152 (*i)->ReMap(oldMap, newMap);
7635093c 153 for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
a9fe5928 154 i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
f7a35f2e 155 (*i)->ReMap(oldMap, newMap);
7635093c 156 for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
a9fe5928 157 i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
f7a35f2e 158 (*i)->ReMap(oldMap, newMap);
7635093c 159 for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
a9fe5928 160 i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
f7a35f2e 161 (*i)->ReMap(oldMap, newMap);
7635093c 162 for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
a9fe5928 163 i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
f7a35f2e 164 (*i)->ReMap(oldMap, newMap);
7635093c 165 for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
a9fe5928 166 i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
f7a35f2e 167 (*i)->ReMap(oldMap, newMap);
7635093c 168 for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
a9fe5928 169 i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
f7a35f2e 170 (*i)->ReMap(oldMap, newMap);
b07aeb1a
DK
171 for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
172 i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
173 (*i)->ReMap(oldMap, newMap);
a9fe5928 174} /*}}}*/
7e58ab0c 175// CacheGenerator::WriteStringInMap /*{{{*/
4ad8619b 176map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
7e58ab0c 177 const unsigned long &Len) {
a9fe5928 178 void const * const oldMap = Map.Data();
4ad8619b 179 map_stringitem_t const index = Map.WriteString(String, Len);
a9fe5928
DK
180 if (index != 0)
181 ReMap(oldMap, Map.Data());
182 return index;
7e58ab0c
DK
183}
184 /*}}}*/
185// CacheGenerator::WriteStringInMap /*{{{*/
4ad8619b 186map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
a9fe5928 187 void const * const oldMap = Map.Data();
4ad8619b 188 map_stringitem_t const index = Map.WriteString(String);
a9fe5928
DK
189 if (index != 0)
190 ReMap(oldMap, Map.Data());
191 return index;
7e58ab0c
DK
192}
193 /*}}}*/
4ad8619b 194map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
a9fe5928 195 void const * const oldMap = Map.Data();
4ad8619b 196 map_pointer_t const index = Map.Allocate(size);
a9fe5928
DK
197 if (index != 0)
198 ReMap(oldMap, Map.Data());
199 return index;
7e58ab0c
DK
200}
201 /*}}}*/
578bfd0a
AL
202// CacheGenerator::MergeList - Merge the package list /*{{{*/
203// ---------------------------------------------------------------------
204/* This provides the generation of the entries in the cache. Each loop
205 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
206bool pkgCacheGenerator::MergeList(ListParser &List,
207 pkgCache::VerIterator *OutVer)
578bfd0a
AL
208{
209 List.Owner = this;
0149949b 210
f9eec0e7 211 unsigned int Counter = 0;
0149949b 212 while (List.Step() == true)
578bfd0a 213 {
5bf15716 214 string const PackageName = List.Package();
65a1e968
AL
215 if (PackageName.empty() == true)
216 return false;
5bf15716 217
99a2ea5a
DK
218 Counter++;
219 if (Counter % 100 == 0 && Progress != 0)
220 Progress->Progress(List.Offset());
221
222 string Arch = List.Architecture();
223 string const Version = List.Version();
224 if (Version.empty() == true && Arch.empty() == true)
225 {
8ec00880 226 // package descriptions
99a2ea5a
DK
227 if (MergeListGroup(List, PackageName) == false)
228 return false;
8ec00880 229 continue;
99a2ea5a
DK
230 }
231
232 if (Arch.empty() == true)
c919ad6e
DK
233 {
234 // use the pseudo arch 'none' for arch-less packages
235 Arch = "none";
236 /* We might built a SingleArchCache here, which we don't want to blow up
237 just for these :none packages to a proper MultiArchCache, so just ensure
238 that we have always a native package structure first for SingleArch */
239 pkgCache::PkgIterator NP;
9abb2283 240 Dynamic<pkgCache::PkgIterator> DynPkg(NP);
c919ad6e
DK
241 if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
242 // TRANSLATOR: The first placeholder is a package name,
243 // the other two should be copied verbatim as they include debug info
244 return _error->Error(_("Error occurred while processing %s (%s%d)"),
245 PackageName.c_str(), "NewPackage", 0);
246 }
99a2ea5a 247
8b32e920 248 // Get a pointer to the package structure
9ddf7030 249 pkgCache::PkgIterator Pkg;
a9fe5928 250 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
28166356 251 if (NewPackage(Pkg, PackageName, Arch) == false)
79e9003c
DK
252 // TRANSLATOR: The first placeholder is a package name,
253 // the other two should be copied verbatim as they include debug info
254 return _error->Error(_("Error occurred while processing %s (%s%d)"),
255 PackageName.c_str(), "NewPackage", 1);
8ce4327b 256
99a2ea5a 257
f55a958f
AL
258 if (Version.empty() == true)
259 {
99a2ea5a
DK
260 if (MergeListPackage(List, Pkg) == false)
261 return false;
262 }
263 else
264 {
265 if (MergeListVersion(List, Pkg, Version, OutVer) == false)
266 return false;
267 }
770c32ec 268
99a2ea5a
DK
269 if (OutVer != 0)
270 {
271 FoundFileDeps |= List.HasFileDeps();
272 return true;
f55a958f 273 }
99a2ea5a
DK
274 }
275
4ad8619b 276 if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
277 return _error->Error(_("Wow, you exceeded the number of package "
278 "names this APT is capable of."));
4ad8619b 279 if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
280 return _error->Error(_("Wow, you exceeded the number of versions "
281 "this APT is capable of."));
4ad8619b 282 if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
283 return _error->Error(_("Wow, you exceeded the number of descriptions "
284 "this APT is capable of."));
4ad8619b 285 if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
286 return _error->Error(_("Wow, you exceeded the number of dependencies "
287 "this APT is capable of."));
288
289 FoundFileDeps |= List.HasFileDeps();
290 return true;
291}
292// CacheGenerator::MergeListGroup /*{{{*/
293bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
294{
295 pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
5f4db009
DK
296 // a group has no data on it's own, only packages have it but these
297 // stanzas like this come from Translation- files to add descriptions,
298 // but without a version we don't need a description for it…
99a2ea5a 299 if (Grp.end() == true)
5f4db009 300 return true;
99a2ea5a
DK
301 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
302
303 pkgCache::PkgIterator Pkg;
304 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
305 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
306 if (MergeListPackage(List, Pkg) == false)
307 return false;
308
309 return true;
310}
311 /*}}}*/
312// CacheGenerator::MergeListPackage /*{{{*/
313bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
314{
315 // we first process the package, then the descriptions
316 // (for deb this package processing is in fact a no-op)
317 pkgCache::VerIterator Ver(Cache);
318 Dynamic<pkgCache::VerIterator> DynVer(Ver);
319 if (List.UsePackage(Pkg, Ver) == false)
79e9003c
DK
320 return _error->Error(_("Error occurred while processing %s (%s%d)"),
321 Pkg.Name(), "UsePackage", 1);
99a2ea5a
DK
322
323 // Find the right version to write the description
324 MD5SumValue CurMd5 = List.Description_md5();
ffe3c68e 325 if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
99359751 326 return true;
ffe3c68e 327 std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
99a2ea5a
DK
328 for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
329 {
a65e22c6 330 pkgCache::DescIterator VerDesc = Ver.DescriptionList();
22f07fc5
DK
331
332 // a version can only have one md5 describing it
a65e22c6 333 if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
22f07fc5 334 continue;
99a2ea5a 335
4ad8619b 336 map_stringitem_t md5idx = VerDesc->md5sum;
ffe3c68e
DK
337 for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
338 {
339 // don't add a new description if we have one for the given
340 // md5 && language
341 if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
342 continue;
a65e22c6 343
ffe3c68e
DK
344 AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
345 }
22f07fc5
DK
346
347 // we can stop here as all "same" versions will share the description
348 break;
99a2ea5a
DK
349 }
350
351 return true;
352}
353 /*}}}*/
354// CacheGenerator::MergeListVersion /*{{{*/
355bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
356 std::string const &Version, pkgCache::VerIterator* &OutVer)
357{
358 pkgCache::VerIterator Ver = Pkg.VersionList();
359 Dynamic<pkgCache::VerIterator> DynVer(Ver);
4ad8619b 360 map_pointer_t *LastVer = &Pkg->VersionList;
99a2ea5a
DK
361 void const * oldMap = Map.Data();
362
68134bda 363 unsigned short const Hash = List.VersionHash();
99a2ea5a
DK
364 if (Ver.end() == false)
365 {
366 /* We know the list is sorted so we use that fact in the search.
367 Insertion of new versions is done with correct sorting */
2246928b 368 int Res = 1;
804de19f 369 for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
578bfd0a 370 {
c24972cb 371 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
7e2b56a3
DK
372 // Version is higher as current version - insert here
373 if (Res > 0)
578bfd0a 374 break;
7e2b56a3 375 // Versionstrings are equal - is hash also equal?
68134bda 376 if (Res == 0 && List.SameVersion(Hash, Ver) == true)
7e2b56a3
DK
377 break;
378 // proceed with the next till we have either the right
379 // or we found another version (which will be lower)
578bfd0a 380 }
7e2b56a3
DK
381
382 /* We already have a version for this item, record that we saw it */
e426a5ff 383 if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
578bfd0a 384 {
f55a958f 385 if (List.UsePackage(Pkg,Ver) == false)
79e9003c
DK
386 return _error->Error(_("Error occurred while processing %s (%s%d)"),
387 Pkg.Name(), "UsePackage", 2);
f78439bf 388
578bfd0a 389 if (NewFileVer(Ver,List) == false)
79e9003c
DK
390 return _error->Error(_("Error occurred while processing %s (%s%d)"),
391 Pkg.Name(), "NewFileVer", 1);
99a2ea5a 392
ddc1d8d0
AL
393 // Read only a single record and return
394 if (OutVer != 0)
395 {
396 *OutVer = Ver;
397 return true;
398 }
99a2ea5a
DK
399
400 return true;
204fbdcc 401 }
99a2ea5a 402 }
204fbdcc 403
99a2ea5a 404 // Add a new version
4ad8619b 405 map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
99a2ea5a 406 if (verindex == 0 && _error->PendingError())
79e9003c
DK
407 return _error->Error(_("Error occurred while processing %s (%s%d)"),
408 Pkg.Name(), "NewVersion", 1);
a9fe5928 409
99a2ea5a 410 if (oldMap != Map.Data())
4ad8619b 411 LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
99a2ea5a 412 *LastVer = verindex;
a52f938b 413
79e9003c
DK
414 if (unlikely(List.NewVersion(Ver) == false))
415 return _error->Error(_("Error occurred while processing %s (%s%d)"),
416 Pkg.Name(), "NewVersion", 2);
a9fe5928 417
79e9003c
DK
418 if (unlikely(List.UsePackage(Pkg,Ver) == false))
419 return _error->Error(_("Error occurred while processing %s (%s%d)"),
420 Pkg.Name(), "UsePackage", 3);
421
422 if (unlikely(NewFileVer(Ver,List) == false))
423 return _error->Error(_("Error occurred while processing %s (%s%d)"),
424 Pkg.Name(), "NewFileVer", 2);
a52f938b 425
5a8e963b
DK
426 pkgCache::GrpIterator Grp = Pkg.Group();
427 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
428
429 /* If it is the first version of this package we need to add implicit
430 Multi-Arch dependencies to all other package versions in the group now -
431 otherwise we just add them for this new version */
432 if (Pkg.VersionList()->NextVer == 0)
433 {
434 pkgCache::PkgIterator P = Grp.PackageList();
435 Dynamic<pkgCache::PkgIterator> DynP(P);
436 for (; P.end() != true; P = Grp.NextPkg(P))
437 {
438 if (P->ID == Pkg->ID)
439 continue;
440 pkgCache::VerIterator V = P.VersionList();
441 Dynamic<pkgCache::VerIterator> DynV(V);
442 for (; V.end() != true; ++V)
443 if (unlikely(AddImplicitDepends(V, Pkg) == false))
444 return _error->Error(_("Error occurred while processing %s (%s%d)"),
445 Pkg.Name(), "AddImplicitDepends", 1);
446 }
c919ad6e
DK
447 /* :none packages are packages without an architecture. They are forbidden by
448 debian-policy, so usually they will only be in (old) dpkg status files -
449 and dpkg will complain about them - and are pretty rare. We therefore do
450 usually not create conflicts while the parent is created, but only if a :none
451 package (= the target) appears. This creates incorrect dependencies on :none
452 for architecture-specific dependencies on the package we copy from, but we
453 will ignore this bug as architecture-specific dependencies are only allowed
454 in jessie and until then the :none packages should be extinct (hopefully).
455 In other words: This should work long enough to allow graceful removal of
456 these packages, it is not supposed to allow users to keep using them … */
457 if (strcmp(Pkg.Arch(), "none") == 0)
458 {
459 pkgCache::PkgIterator M = Grp.FindPreferredPkg();
460 if (M.end() == false && Pkg != M)
461 {
462 pkgCache::DepIterator D = M.RevDependsList();
463 Dynamic<pkgCache::DepIterator> DynD(D);
464 for (; D.end() == false; ++D)
465 {
466 if ((D->Type != pkgCache::Dep::Conflicts &&
467 D->Type != pkgCache::Dep::DpkgBreaks &&
468 D->Type != pkgCache::Dep::Replaces) ||
469 D.ParentPkg().Group() == Grp)
470 continue;
471
4ad8619b 472 map_pointer_t *OldDepLast = NULL;
c919ad6e 473 pkgCache::VerIterator ConVersion = D.ParentVer();
9abb2283 474 Dynamic<pkgCache::VerIterator> DynV(ConVersion);
c919ad6e 475 // duplicate the Conflicts/Breaks/Replaces for :none arch
9c44383f
DK
476 NewDepends(Pkg, ConVersion, D->Version,
477 D->CompareOp, D->Type, OldDepLast);
c919ad6e
DK
478 }
479 }
480 }
5a8e963b
DK
481 }
482 if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
483 return _error->Error(_("Error occurred while processing %s (%s%d)"),
484 Pkg.Name(), "AddImplicitDepends", 2);
a52f938b 485
99a2ea5a
DK
486 // Read only a single record and return
487 if (OutVer != 0)
488 {
489 *OutVer = Ver;
490 return true;
578bfd0a 491 }
0149949b 492
ffe3c68e 493 /* Record the Description(s) based on their master md5sum */
5f4db009 494 MD5SumValue CurMd5 = List.Description_md5();
ffe3c68e 495 if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
5f4db009 496 return true;
5f4db009
DK
497
498 /* Before we add a new description we first search in the group for
499 a version with a description of the same MD5 - if so we reuse this
500 description group instead of creating our own for this version */
5f4db009
DK
501 for (pkgCache::PkgIterator P = Grp.PackageList();
502 P.end() == false; P = Grp.NextPkg(P))
503 {
504 for (pkgCache::VerIterator V = P.VersionList();
505 V.end() == false; ++V)
506 {
ffe3c68e 507 if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
5f4db009
DK
508 continue;
509 Ver->DescriptionList = V->DescriptionList;
5f4db009
DK
510 }
511 }
512
ffe3c68e 513 // We haven't found reusable descriptions, so add the first description(s)
4ad8619b 514 map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
ffe3c68e
DK
515 std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
516 for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
517 if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
518 return false;
519 return true;
520}
521 /*}}}*/
4ad8619b 522bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
ffe3c68e
DK
523{
524 pkgCache::DescIterator Desc;
99a2ea5a 525 Dynamic<pkgCache::DescIterator> DynDesc(Desc);
99a2ea5a 526
4ad8619b 527 map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
5954d4d2
DK
528 if (unlikely(descindex == 0 && _error->PendingError()))
529 return _error->Error(_("Error occurred while processing %s (%s%d)"),
ffe3c68e
DK
530 Ver.ParentPkg().Name(), "NewDescription", 1);
531
532 md5idx = Desc->md5sum;
533 Desc->ParentPkg = Ver.ParentPkg().Index();
a65e22c6 534
ffe3c68e
DK
535 // we add at the end, so that the start is constant as we need
536 // that to be able to efficiently share these lists
537 pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
538 for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
4ad8619b 539 map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
ffe3c68e 540 *LastNextDesc = descindex;
99a2ea5a 541
a65e22c6 542 if (NewFileDesc(Desc,List) == false)
79e9003c 543 return _error->Error(_("Error occurred while processing %s (%s%d)"),
ffe3c68e 544 Ver.ParentPkg().Name(), "NewFileDesc", 1);
45415543 545
578bfd0a
AL
546 return true;
547}
548 /*}}}*/
99a2ea5a 549 /*}}}*/
45415543
AL
550// CacheGenerator::MergeFileProvides - Merge file provides /*{{{*/
551// ---------------------------------------------------------------------
552/* If we found any file depends while parsing the main list we need to
553 resolve them. Since it is undesired to load the entire list of files
554 into the cache as virtual packages we do a two stage effort. MergeList
555 identifies the file depends and this creates Provdies for them by
556 re-parsing all the indexs. */
557bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
558{
559 List.Owner = this;
560
561 unsigned int Counter = 0;
562 while (List.Step() == true)
563 {
564 string PackageName = List.Package();
565 if (PackageName.empty() == true)
566 return false;
567 string Version = List.Version();
568 if (Version.empty() == true)
569 continue;
570
571 pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
a9fe5928 572 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
45415543 573 if (Pkg.end() == true)
79e9003c
DK
574 return _error->Error(_("Error occurred while processing %s (%s%d)"),
575 PackageName.c_str(), "FindPkg", 1);
45415543
AL
576 Counter++;
577 if (Counter % 100 == 0 && Progress != 0)
578 Progress->Progress(List.Offset());
579
68134bda 580 unsigned short Hash = List.VersionHash();
45415543 581 pkgCache::VerIterator Ver = Pkg.VersionList();
a9fe5928 582 Dynamic<pkgCache::VerIterator> DynVer(Ver);
f7f0d6c7 583 for (; Ver.end() == false; ++Ver)
45415543 584 {
68134bda 585 if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
45415543
AL
586 {
587 if (List.CollectFileProvides(Cache,Ver) == false)
79e9003c
DK
588 return _error->Error(_("Error occurred while processing %s (%s%d)"),
589 PackageName.c_str(), "CollectFileProvides", 1);
45415543
AL
590 break;
591 }
592 }
593
594 if (Ver.end() == true)
595 _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
596 }
597
598 return true;
599}
600 /*}}}*/
5bf15716
DK
601// CacheGenerator::NewGroup - Add a new group /*{{{*/
602// ---------------------------------------------------------------------
603/* This creates a new group structure and adds it to the hash table */
33dd02e3
DK
604bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
605{
606 Grp = Cache.FindGrp(Name);
607 if (Grp.end() == false)
608 return true;
5bf15716 609
33dd02e3 610 // Get a structure
4ad8619b 611 map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
33dd02e3
DK
612 if (unlikely(Group == 0))
613 return false;
5bf15716 614
33dd02e3 615 Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
b13ce62c 616 map_stringitem_t const idxName = StoreString(PKGNAME, Name);
a9fe5928 617 if (unlikely(idxName == 0))
33dd02e3 618 return false;
a9fe5928 619 Grp->Name = idxName;
5bf15716 620
33dd02e3
DK
621 // Insert it into the hash table
622 unsigned long const Hash = Cache.Hash(Name);
32ab4bd0 623 map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
aa0fe657
DK
624 while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
625 insertAt = &(Cache.GrpP + *insertAt)->Next;
626 Grp->Next = *insertAt;
627 *insertAt = Group;
5bf15716 628
52c41485 629 Grp->ID = Cache.HeaderP->GroupCount++;
33dd02e3 630 return true;
5bf15716
DK
631}
632 /*}}}*/
578bfd0a
AL
633// CacheGenerator::NewPackage - Add a new package /*{{{*/
634// ---------------------------------------------------------------------
635/* This creates a new package structure and adds it to the hash table */
5bf15716
DK
636bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
637 const string &Arch) {
638 pkgCache::GrpIterator Grp;
a9fe5928 639 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
5bf15716
DK
640 if (unlikely(NewGroup(Grp, Name) == false))
641 return false;
642
643 Pkg = Grp.FindPkg(Arch);
644 if (Pkg.end() == false)
645 return true;
a52f938b 646
578bfd0a 647 // Get a structure
4ad8619b 648 map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
5bf15716 649 if (unlikely(Package == 0))
578bfd0a 650 return false;
f55a958f 651 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
5bf15716 652
ecc138f8
DK
653 // Set the name, arch and the ID
654 APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
655 Pkg->Group = Grp.Index();
656 // all is mapped to the native architecture
657 map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
658 if (unlikely(idxArch == 0))
659 return false;
660 Pkg->Arch = idxArch;
661 Pkg->ID = Cache.HeaderP->PackageCount++;
662
c408e01e
DK
663 // Insert the package into our package list
664 if (Grp->FirstPackage == 0) // the group is new
665 {
aa0fe657 666 Grp->FirstPackage = Package;
c408e01e 667 // Insert it into the hash table
4ad8619b 668 map_id_t const Hash = Cache.Hash(Name);
32ab4bd0 669 map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
fe86debb 670 while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
32ab4bd0
DK
671 insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
672 Pkg->NextPackage = *insertAt;
aa0fe657 673 *insertAt = Package;
c408e01e
DK
674 }
675 else // Group the Packages together
676 {
ecc138f8
DK
677 // but first get implicit provides done
678 if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
679 {
680 pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
681 if (M.end() == false)
682 for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
683 {
684 if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
685 continue;
686 pkgCache::VerIterator Ver = Prv.OwnerVer();
687 if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
688 ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
689 (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
690 if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
691 return false;
692 }
693
694 for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
695 for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
696 if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
697 if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
698 return false;
699 }
bb0f6a34
DK
700 // and negative dependencies, don't forget negative dependencies
701 {
702 pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
703 if (M.end() == false)
704 for (pkgCache::DepIterator Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
705 {
706 if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
707 continue;
708 if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
709 Dep->Type != pkgCache::Dep::Replaces)
710 continue;
711 pkgCache::VerIterator Ver = Dep.ParentVer();
712 map_pointer_t * unused = NULL;
713 if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
714 return false;
715 }
716 }
ecc138f8 717
c408e01e
DK
718 // this package is the new last package
719 pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
32ab4bd0
DK
720 Pkg->NextPackage = LastPkg->NextPackage;
721 LastPkg->NextPackage = Package;
c408e01e
DK
722 }
723 Grp->LastPackage = Package;
578bfd0a
AL
724 return true;
725}
5a8e963b
DK
726 /*}}}*/
727// CacheGenerator::AddImplicitDepends /*{{{*/
728bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
729 pkgCache::PkgIterator &P,
730 pkgCache::VerIterator &V)
731{
732 // copy P.Arch() into a string here as a cache remap
733 // in NewDepends() later may alter the pointer location
734 string Arch = P.Arch() == NULL ? "" : P.Arch();
4ad8619b 735 map_pointer_t *OldDepLast = NULL;
5a8e963b
DK
736 /* MultiArch handling introduces a lot of implicit Dependencies:
737 - MultiArch: same → Co-Installable if they have the same version
738 - All others conflict with all other group members */
739 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
740 pkgCache::PkgIterator D = G.PackageList();
741 Dynamic<pkgCache::PkgIterator> DynD(D);
4ad8619b 742 map_stringitem_t const VerStrIdx = V->VerStr;
5a8e963b
DK
743 for (; D.end() != true; D = G.NextPkg(D))
744 {
745 if (Arch == D.Arch() || D->VersionList == 0)
746 continue;
747 /* We allow only one installed arch at the time
748 per group, therefore each group member conflicts
749 with all other group members */
750 if (coInstall == true)
751 {
752 // Replaces: ${self}:other ( << ${binary:Version})
9c44383f 753 NewDepends(D, V, VerStrIdx,
8c7af4d4 754 pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
5a8e963b
DK
755 OldDepLast);
756 // Breaks: ${self}:other (!= ${binary:Version})
9c44383f 757 NewDepends(D, V, VerStrIdx,
8c7af4d4 758 pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
5a8e963b
DK
759 OldDepLast);
760 } else {
761 // Conflicts: ${self}:other
9c44383f 762 NewDepends(D, V, 0,
8c7af4d4 763 pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
5a8e963b
DK
764 OldDepLast);
765 }
766 }
767 return true;
768}
769bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
770 pkgCache::PkgIterator &D)
771{
772 /* MultiArch handling introduces a lot of implicit Dependencies:
773 - MultiArch: same → Co-Installable if they have the same version
774 - All others conflict with all other group members */
4ad8619b 775 map_pointer_t *OldDepLast = NULL;
5a8e963b
DK
776 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
777 if (coInstall == true)
778 {
4ad8619b 779 map_stringitem_t const VerStrIdx = V->VerStr;
5a8e963b 780 // Replaces: ${self}:other ( << ${binary:Version})
9c44383f 781 NewDepends(D, V, VerStrIdx,
8c7af4d4 782 pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
5a8e963b
DK
783 OldDepLast);
784 // Breaks: ${self}:other (!= ${binary:Version})
9c44383f 785 NewDepends(D, V, VerStrIdx,
8c7af4d4 786 pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
5a8e963b
DK
787 OldDepLast);
788 } else {
789 // Conflicts: ${self}:other
9c44383f 790 NewDepends(D, V, 0,
8c7af4d4 791 pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
5a8e963b
DK
792 OldDepLast);
793 }
794 return true;
795}
796
578bfd0a
AL
797 /*}}}*/
798// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
799// ---------------------------------------------------------------------
800/* */
f55a958f 801bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
802 ListParser &List)
803{
ddc1d8d0
AL
804 if (CurrentFile == 0)
805 return true;
806
dcb79bae 807 // Get a structure
4ad8619b 808 map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
dcb79bae 809 if (VerFile == 0)
ac2c559b 810 return false;
dcb79bae
AL
811
812 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
813 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
814
815 // Link it to the end of the list
4ad8619b 816 map_pointer_t *Last = &Ver->FileList;
f7f0d6c7 817 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
03e39e59
AL
818 Last = &V->NextFile;
819 VF->NextFile = *Last;
820 *Last = VF.Index();
821
dcb79bae
AL
822 VF->Offset = List.Offset();
823 VF->Size = List.Size();
ad00ae81
AL
824 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
825 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
826 Cache.HeaderP->VerFileCount++;
827
f55a958f 828 return true;
578bfd0a
AL
829}
830 /*}}}*/
831// CacheGenerator::NewVersion - Create a new Version /*{{{*/
832// ---------------------------------------------------------------------
f55a958f 833/* This puts a version structure in the linked list */
4ad8619b 834map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
171c75f1 835 const string &VerStr,
4ad8619b
DK
836 map_pointer_t const ParentPkg,
837 unsigned short const Hash,
838 map_pointer_t const Next)
578bfd0a 839{
f55a958f 840 // Get a structure
4ad8619b 841 map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
f55a958f 842 if (Version == 0)
0149949b 843 return 0;
f55a958f
AL
844
845 // Fill it in
846 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
dfe45e1f 847 //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
f55a958f 848 Ver->NextVer = Next;
885594fc
DK
849 Ver->ParentPkg = ParentPkg;
850 Ver->Hash = Hash;
f55a958f 851 Ver->ID = Cache.HeaderP->VersionCount++;
885594fc
DK
852
853 // try to find the version string in the group for reuse
854 pkgCache::PkgIterator Pkg = Ver.ParentPkg();
855 pkgCache::GrpIterator Grp = Pkg.Group();
856 if (Pkg.end() == false && Grp.end() == false)
857 {
858 for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
859 {
860 if (Pkg == P)
861 continue;
862 for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
863 {
864 int const cmp = strcmp(V.VerStr(), VerStr.c_str());
865 if (cmp == 0)
866 {
867 Ver->VerStr = V->VerStr;
868 return Version;
869 }
870 else if (cmp < 0)
871 break;
872 }
873 }
874 }
875 // haven't found the version string, so create
30b45652 876 map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
a9fe5928 877 if (unlikely(idxVerStr == 0))
0149949b 878 return 0;
a9fe5928 879 Ver->VerStr = idxVerStr;
0149949b 880 return Version;
578bfd0a
AL
881}
882 /*}}}*/
a52f938b
OS
883// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
884// ---------------------------------------------------------------------
885/* */
886bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
887 ListParser &List)
888{
889 if (CurrentFile == 0)
890 return true;
891
892 // Get a structure
4ad8619b 893 map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
a52f938b 894 if (DescFile == 0)
c5f44afc 895 return false;
770c32ec 896
a52f938b
OS
897 pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
898 DF->File = CurrentFile - Cache.PkgFileP;
770c32ec 899
a52f938b 900 // Link it to the end of the list
4ad8619b 901 map_pointer_t *Last = &Desc->FileList;
f7f0d6c7 902 for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
a52f938b 903 Last = &D->NextFile;
770c32ec 904
a52f938b
OS
905 DF->NextFile = *Last;
906 *Last = DF.Index();
907
908 DF->Offset = List.Offset();
909 DF->Size = List.Size();
910 if (Cache.HeaderP->MaxDescFileSize < DF->Size)
911 Cache.HeaderP->MaxDescFileSize = DF->Size;
912 Cache.HeaderP->DescFileCount++;
913
914 return true;
915}
916 /*}}}*/
917// CacheGenerator::NewDescription - Create a new Description /*{{{*/
918// ---------------------------------------------------------------------
919/* This puts a description structure in the linked list */
4ad8619b 920map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
a65e22c6
DK
921 const string &Lang,
922 const MD5SumValue &md5sum,
4ad8619b 923 map_stringitem_t const idxmd5str)
a52f938b
OS
924{
925 // Get a structure
4ad8619b 926 map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
a52f938b
OS
927 if (Description == 0)
928 return 0;
929
930 // Fill it in
931 Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
a52f938b 932 Desc->ID = Cache.HeaderP->DescriptionCount++;
78a5476f 933 map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
a65e22c6 934 if (unlikely(idxlanguage_code == 0))
c5f44afc 935 return 0;
a9fe5928 936 Desc->language_code = idxlanguage_code;
a65e22c6
DK
937
938 if (idxmd5str != 0)
939 Desc->md5sum = idxmd5str;
940 else
941 {
4ad8619b 942 map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
a65e22c6
DK
943 if (unlikely(idxmd5sum == 0))
944 return 0;
945 Desc->md5sum = idxmd5sum;
946 }
a52f938b
OS
947
948 return Description;
949}
950 /*}}}*/
25396fb0 951// CacheGenerator::NewDepends - Create a dependency element /*{{{*/
dcb79bae
AL
952// ---------------------------------------------------------------------
953/* This creates a dependency element in the tree. It is linked to the
954 version and to the package that it is pointing to. */
9c44383f
DK
955bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
956 pkgCache::VerIterator &Ver,
4ad8619b 957 map_pointer_t const Version,
8c7af4d4
DK
958 uint8_t const Op,
959 uint8_t const Type,
4ad8619b 960 map_pointer_t* &OldDepLast)
dcb79bae 961{
a9fe5928 962 void const * const oldMap = Map.Data();
dcb79bae 963 // Get a structure
4ad8619b 964 map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
5bf15716 965 if (unlikely(Dependency == 0))
dcb79bae 966 return false;
9c44383f 967
71c9e95b
DK
968 bool isDuplicate = false;
969 map_pointer_t DependencyData = 0;
b291aa59
DK
970 map_pointer_t PreviousData = 0;
971 if (Pkg->RevDepends != 0)
71c9e95b 972 {
b291aa59
DK
973 pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
974 DependencyData = L->DependencyData;
975 do {
976 pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
977 if (Version > D->Version)
978 break;
979 if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
980 {
981 isDuplicate = true;
982 break;
983 }
984 PreviousData = DependencyData;
985 DependencyData = D->NextData;
986 } while (DependencyData != 0);
71c9e95b
DK
987 }
988
989 if (isDuplicate == false)
990 {
71c9e95b
DK
991 DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
992 if (unlikely(DependencyData == 0))
993 return false;
71c9e95b 994 }
5bf15716 995
71c9e95b
DK
996 pkgCache::Dependency * Link = Cache.DepP + Dependency;
997 Link->ParentVer = Ver.Index();
998 Link->DependencyData = DependencyData;
999 Link->ID = Cache.HeaderP->DependsCount++;
1000
1001 pkgCache::DepIterator Dep(Cache, Link);
71c9e95b
DK
1002 if (isDuplicate == false)
1003 {
1004 Dep->Type = Type;
1005 Dep->CompareOp = Op;
1006 Dep->Version = Version;
1007 Dep->Package = Pkg.Index();
1008 ++Cache.HeaderP->DependsDataCount;
b291aa59
DK
1009 if (PreviousData != 0)
1010 {
1011 pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1012 Dep->NextData = D->NextData;
1013 D->NextData = DependencyData;
1014 }
1015 else if (Pkg->RevDepends != 0)
1016 {
1017 pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1018 Dep->NextData = D->DependencyData;
1019 }
1020 }
1021
1022 if (isDuplicate == true || PreviousData != 0)
1023 {
1024 pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1025 Link->NextRevDepends = L->NextRevDepends;
1026 L->NextRevDepends = Dependency;
71c9e95b
DK
1027 }
1028 else
1029 {
b291aa59
DK
1030 Link->NextRevDepends = Pkg->RevDepends;
1031 Pkg->RevDepends = Dependency;
71c9e95b 1032 }
25396fb0 1033
b291aa59 1034
25396fb0
DK
1035 // Do we know where to link the Dependency to?
1036 if (OldDepLast == NULL)
c1a22377 1037 {
f9eec0e7 1038 OldDepLast = &Ver->DependsList;
f7f0d6c7 1039 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
f9eec0e7 1040 OldDepLast = &D->NextDepends;
a9fe5928 1041 } else if (oldMap != Map.Data())
4ad8619b 1042 OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
45415543 1043
f9eec0e7 1044 Dep->NextDepends = *OldDepLast;
71c9e95b 1045 *OldDepLast = Dependency;
f9eec0e7 1046 OldDepLast = &Dep->NextDepends;
dcb79bae
AL
1047 return true;
1048}
1049 /*}}}*/
25396fb0
DK
1050// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1051// ---------------------------------------------------------------------
1052/* This creates a Group and the Package to link this dependency to if
1053 needed and handles also the caching of the old endpoint */
32b9a14c 1054bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
25396fb0
DK
1055 const string &PackageName,
1056 const string &Arch,
1057 const string &Version,
8c7af4d4
DK
1058 uint8_t const Op,
1059 uint8_t const Type)
25396fb0
DK
1060{
1061 pkgCache::GrpIterator Grp;
a9fe5928 1062 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
25396fb0
DK
1063 if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1064 return false;
1065
25396fb0
DK
1066 // Is it a file dependency?
1067 if (unlikely(PackageName[0] == '/'))
1068 FoundFileDeps = true;
1069
bb0f6a34
DK
1070 map_stringitem_t idxVersion = 0;
1071 if (Version.empty() == false)
1072 {
1073 int const CmpOp = Op & 0x0F;
1074 // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1075 if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
1076 idxVersion = Ver->VerStr;
1077
1078 if (idxVersion == 0)
1079 {
1080 idxVersion = StoreString(VERSIONNUMBER, Version);
1081 if (unlikely(idxVersion == 0))
1082 return false;
1083 }
25396fb0
DK
1084 }
1085
bb0f6a34
DK
1086 bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1087 Type == pkgCache::Dep::Conflicts ||
1088 Type == pkgCache::Dep::Replaces);
1089
1090 pkgCache::PkgIterator Pkg;
1091 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1092 if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1093 {
1094 // Locate the target package
1095 Pkg = Grp.FindPkg(Arch);
1096 if (Pkg.end() == true) {
1097 if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1098 return false;
1099 }
1100
1101 /* Caching the old end point speeds up generation substantially */
1102 if (OldDepVer != Ver) {
1103 OldDepLast = NULL;
1104 OldDepVer = Ver;
1105 }
1106
1107 return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1108 }
1109 else
1110 {
1111 /* Caching the old end point speeds up generation substantially */
1112 if (OldDepVer != Ver) {
1113 OldDepLast = NULL;
1114 OldDepVer = Ver;
1115 }
1116
1117 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1118 {
1119 if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1120 return false;
1121 }
1122 }
1123 return true;
25396fb0
DK
1124}
1125 /*}}}*/
dcb79bae 1126// ListParser::NewProvides - Create a Provides element /*{{{*/
32b9a14c 1127bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
8c7af4d4 1128 const string &PkgName,
67e0766f 1129 const string &PkgArch,
8c7af4d4
DK
1130 const string &Version,
1131 uint8_t const Flags)
dcb79bae 1132{
ecc138f8 1133 pkgCache const &Cache = Owner->Cache;
8efa2a3b
AL
1134
1135 // We do not add self referencing provides
566046f4 1136 if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
959470da 1137 (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
8efa2a3b 1138 return true;
ecc138f8
DK
1139
1140 // Locate the target package
1141 pkgCache::PkgIterator Pkg;
1142 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1143 if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1144 return false;
1145
1146 map_stringitem_t idxProvideVersion = 0;
1147 if (Version.empty() == false) {
1148 idxProvideVersion = StoreString(VERSIONNUMBER, Version);
1149 if (unlikely(idxProvideVersion == 0))
1150 return false;
1151 }
1152 return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1153}
1154bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1155 pkgCache::PkgIterator &Pkg,
1156 map_pointer_t const ProvideVersion,
1157 uint8_t const Flags)
1158{
dcb79bae 1159 // Get a structure
ecc138f8 1160 map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
5bf15716 1161 if (unlikely(Provides == 0))
dcb79bae 1162 return false;
ecc138f8
DK
1163 ++Cache.HeaderP->ProvidesCount;
1164
dcb79bae
AL
1165 // Fill it in
1166 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1167 Prv->Version = Ver.Index();
ecc138f8 1168 Prv->ProvideVersion = ProvideVersion;
8c7af4d4 1169 Prv->Flags = Flags;
dcb79bae
AL
1170 Prv->NextPkgProv = Ver->ProvidesList;
1171 Ver->ProvidesList = Prv.Index();
ecc138f8 1172
dcb79bae
AL
1173 // Link it to the package
1174 Prv->ParentPkg = Pkg.Index();
1175 Prv->NextProvides = Pkg->ProvidesList;
1176 Pkg->ProvidesList = Prv.Index();
ecc138f8
DK
1177 return true;
1178}
1179 /*}}}*/
1180// ListParser::NewProvidesAllArch - add provides for all architectures /*{{{*/
1181bool pkgCacheGenerator::ListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1182 string const &Version, uint8_t const Flags) {
1183 pkgCache &Cache = Owner->Cache;
1184 pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
1185 if (Grp.end() == true)
1186 return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1187 else
1188 {
1189 map_stringitem_t idxProvideVersion = 0;
1190 if (Version.empty() == false) {
1191 idxProvideVersion = StoreString(VERSIONNUMBER, Version);
1192 if (unlikely(idxProvideVersion == 0))
1193 return false;
1194 }
1195
1196 bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1197 bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1198 pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
1199 for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1200 {
1201 if (isImplicit && OwnerPkg == Pkg)
1202 continue;
1203 if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1204 continue;
1205 if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1206 return false;
1207 }
1208 }
dcb79bae
AL
1209 return true;
1210}
1211 /*}}}*/
68134bda
DK
1212bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1213 pkgCache::VerIterator const &Ver)
1214{
1215 return Hash == Ver->Hash;
1216}
1217 /*}}}*/
b07aeb1a
DK
1218// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to /*{{{*/
1219bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1220 unsigned long Flags)
1221{
1222 if (File.empty() && Site.empty())
1223 {
1224 CurrentRlsFile = NULL;
1225 return true;
1226 }
1227
1228 // Get some space for the structure
1229 map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1230 if (unlikely(idxFile == 0))
1231 return false;
1232 CurrentRlsFile = Cache.RlsFileP + idxFile;
1233
1234 // Fill it in
1235 map_stringitem_t const idxFileName = WriteStringInMap(File);
1236 map_stringitem_t const idxSite = StoreString(MIXED, Site);
1237 if (unlikely(idxFileName == 0 || idxSite == 0))
1238 return false;
1239 CurrentRlsFile->FileName = idxFileName;
1240 CurrentRlsFile->Site = idxSite;
1241 CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1242 CurrentRlsFile->Flags = Flags;
1243 CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1244 RlsFileName = File;
1245 Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1246 Cache.HeaderP->ReleaseFileCount++;
1247
1248 return true;
1249}
1250 /*}}}*/
578bfd0a
AL
1251// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
1252// ---------------------------------------------------------------------
1253/* This is used to select which file is to be associated with all newly
b2e465d6 1254 added versions. The caller is responsible for setting the IMS fields. */
b07aeb1a
DK
1255bool pkgCacheGenerator::SelectFile(std::string const &File,
1256 pkgIndexFile const &Index,
e185d8b3 1257 std::string const &Architecture,
b07aeb1a
DK
1258 std::string const &Component,
1259 unsigned long const Flags)
578bfd0a 1260{
578bfd0a 1261 // Get some space for the structure
4ad8619b 1262 map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
a9fe5928 1263 if (unlikely(idxFile == 0))
578bfd0a 1264 return false;
a9fe5928
DK
1265 CurrentFile = Cache.PkgFileP + idxFile;
1266
578bfd0a 1267 // Fill it in
4ad8619b 1268 map_stringitem_t const idxFileName = WriteStringInMap(File);
b07aeb1a 1269 if (unlikely(idxFileName == 0))
a9fe5928
DK
1270 return false;
1271 CurrentFile->FileName = idxFileName;
578bfd0a 1272 CurrentFile->NextFile = Cache.HeaderP->FileList;
e1b74f61 1273 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
78a5476f 1274 map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
a9fe5928
DK
1275 if (unlikely(idxIndexType == 0))
1276 return false;
1277 CurrentFile->IndexType = idxIndexType;
e185d8b3
DK
1278 if (Architecture.empty())
1279 CurrentFile->Architecture = 0;
1280 else
1281 {
1282 map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1283 if (unlikely(arch == 0))
1284 return false;
1285 CurrentFile->Architecture = arch;
1286 }
b07aeb1a
DK
1287 map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1288 if (unlikely(component == 0))
1289 return false;
1290 CurrentFile->Component = component;
1291 CurrentFile->Flags = Flags;
1292 if (CurrentRlsFile != NULL)
1293 CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1294 else
1295 CurrentFile->Release = 0;
578bfd0a 1296 PkgFileName = File;
ad00ae81 1297 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 1298 Cache.HeaderP->PackageFileCount++;
b2e465d6 1299
ddc1d8d0 1300 if (Progress != 0)
b2e465d6 1301 Progress->SubProgress(Index.Size());
8efa2a3b 1302 return true;
578bfd0a
AL
1303}
1304 /*}}}*/
f55a958f
AL
1305// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
1306// ---------------------------------------------------------------------
1307/* This is used to create handles to strings. Given the same text it
1308 always returns the same number */
78a5476f 1309map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
f55a958f
AL
1310 unsigned int Size)
1311{
78a5476f
DK
1312 std::string const key(S, Size);
1313
1314 std::map<std::string,map_stringitem_t> * strings;
1315 switch(type) {
1316 case MIXED: strings = &strMixed; break;
1317 case PKGNAME: strings = &strPkgNames; break;
30b45652 1318 case VERSIONNUMBER: strings = &strVersions; break;
78a5476f
DK
1319 case SECTION: strings = &strSections; break;
1320 default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
f55a958f 1321 }
a9fe5928 1322
78a5476f
DK
1323 std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1324 if (item != strings->end())
1325 return item->second;
a9fe5928 1326
78a5476f
DK
1327 map_stringitem_t const idxString = WriteStringInMap(S,Size);
1328 strings->insert(std::make_pair(key, idxString));
1329 return idxString;
f55a958f
AL
1330}
1331 /*}}}*/
b2e465d6 1332// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 1333// ---------------------------------------------------------------------
b2e465d6
AL
1334/* This just verifies that each file in the list of index files exists,
1335 has matching attributes with the cache and the cache does not have
1336 any extra files. */
5465192b 1337static bool CheckValidity(const string &CacheFile,
2ec858bc 1338 pkgSourceList &List,
5465192b
DK
1339 FileIterator const Start,
1340 FileIterator const End,
2ec858bc 1341 MMap **OutMap = 0)
b35d2f5f 1342{
c8e572e3 1343 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
b2e465d6
AL
1344 // No file, certainly invalid
1345 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
c8e572e3
MV
1346 {
1347 if (Debug == true)
5465192b 1348 std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
b35d2f5f 1349 return false;
c8e572e3
MV
1350 }
1351
96d14a91 1352 if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
2ec858bc
MV
1353 {
1354 if (Debug == true)
1355 std::clog << "sources.list is newer than the cache" << std::endl;
1356 return false;
1357 }
1358
b2e465d6 1359 // Map it
b35d2f5f 1360 FileFd CacheF(CacheFile,FileFd::ReadOnly);
eb162ff7 1361 SPtr<MMap> Map = new MMap(CacheF,0);
b35d2f5f 1362 pkgCache Cache(Map);
b2e465d6 1363 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f 1364 {
c8e572e3 1365 if (Debug == true)
5465192b 1366 std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
b35d2f5f
AL
1367 _error->Discard();
1368 return false;
1369 }
b07aeb1a
DK
1370
1371 SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1372 memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1373 std::vector<pkgIndexFile *> Files;
1374 for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1375 {
1376 if (Debug == true)
1377 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
3fd89e62 1378 pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
b07aeb1a
DK
1379 if (RlsFile.end() == true)
1380 {
1381 if (Debug == true)
1382 std::clog << "FindInCache returned end-Pointer" << std::endl;
1383 return false;
1384 }
1385
1386 RlsVisited[RlsFile->ID] = true;
1387 if (Debug == true)
1388 std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1389
5465192b
DK
1390 std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1391 std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1392 [](pkgIndexFile const * const I) { return I->HasPackages(); });
b07aeb1a
DK
1393 }
1394 for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1395 if (RlsVisited[I] == false)
1396 {
1397 if (Debug == true)
1398 std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1399 return false;
1400 }
1401
5465192b 1402 std::copy(Start, End, std::back_inserter(Files));
b07aeb1a 1403
b2e465d6
AL
1404 /* Now we check every index file, see if it is in the cache,
1405 verify the IMS data and check that it is on the disk too.. */
1406 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1407 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
b07aeb1a 1408 for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
c8e572e3
MV
1409 {
1410 if (Debug == true)
b07aeb1a
DK
1411 std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1412 if ((*PkgFile)->Exists() == false)
b35d2f5f 1413 {
c8e572e3
MV
1414 if (Debug == true)
1415 std::clog << "file doesn't exist" << std::endl;
b2e465d6 1416 continue;
b35d2f5f 1417 }
b2e465d6
AL
1418
1419 // FindInCache is also expected to do an IMS check.
b07aeb1a 1420 pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
b2e465d6 1421 if (File.end() == true)
c8e572e3
MV
1422 {
1423 if (Debug == true)
1424 std::clog << "FindInCache returned end-Pointer" << std::endl;
b35d2f5f 1425 return false;
c8e572e3 1426 }
a52f938b 1427
b2e465d6 1428 Visited[File->ID] = true;
c8e572e3
MV
1429 if (Debug == true)
1430 std::clog << "with ID " << File->ID << " is valid" << std::endl;
b35d2f5f 1431 }
b07aeb1a 1432
b2e465d6
AL
1433 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1434 if (Visited[I] == false)
c8e572e3
MV
1435 {
1436 if (Debug == true)
b07aeb1a 1437 std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
b2e465d6 1438 return false;
c8e572e3 1439 }
b07aeb1a 1440
b35d2f5f
AL
1441 if (_error->PendingError() == true)
1442 {
c8e572e3
MV
1443 if (Debug == true)
1444 {
1445 std::clog << "Validity failed because of pending errors:" << std::endl;
1446 _error->DumpErrors();
1447 }
b35d2f5f
AL
1448 _error->Discard();
1449 return false;
1450 }
b35d2f5f 1451
b2e465d6
AL
1452 if (OutMap != 0)
1453 *OutMap = Map.UnGuard();
b35d2f5f
AL
1454 return true;
1455}
1456 /*}}}*/
b2e465d6 1457// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 1458// ---------------------------------------------------------------------
b2e465d6
AL
1459/* Size is kind of an abstract notion that is only used for the progress
1460 meter */
b07aeb1a 1461static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
b35d2f5f 1462{
4ad8619b 1463 map_filesize_t TotalSize = 0;
b07aeb1a
DK
1464 if (List != NULL)
1465 {
1466 for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1467 {
1468 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1469 for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1470 if ((*j)->HasPackages() == true)
1471 TotalSize += (*j)->Size();
1472 }
1473 }
1474
ef74268b 1475 for (; Start < End; ++Start)
b35d2f5f 1476 {
b2e465d6 1477 if ((*Start)->HasPackages() == false)
ef74268b 1478 continue;
b2e465d6 1479 TotalSize += (*Start)->Size();
b35d2f5f 1480 }
b2e465d6 1481 return TotalSize;
2d11135a
AL
1482}
1483 /*}}}*/
b2e465d6 1484// BuildCache - Merge the list of index files into the cache /*{{{*/
b2e465d6 1485static bool BuildCache(pkgCacheGenerator &Gen,
5465192b 1486 OpProgress * const Progress,
4ad8619b 1487 map_filesize_t &CurrentSize,map_filesize_t TotalSize,
b07aeb1a 1488 pkgSourceList const * const List,
5465192b 1489 FileIterator const Start, FileIterator const End)
2d11135a 1490{
b07aeb1a
DK
1491 std::vector<pkgIndexFile *> Files;
1492 bool const HasFileDeps = Gen.HasFileDeps();
5465192b
DK
1493 bool mergeFailure = false;
1494
1495 auto const indexFileMerge = [&](pkgIndexFile * const I) {
1496 if (HasFileDeps)
1497 Files.push_back(I);
1498
1499 if (I->HasPackages() == false || mergeFailure)
1500 return;
1501
1502 if (I->Exists() == false)
1503 return;
1504
1505 if (I->FindInCache(Gen.GetCache()).end() == false)
1506 {
1507 _error->Warning("Duplicate sources.list entry %s",
1508 I->Describe().c_str());
1509 return;
1510 }
1511
1512 map_filesize_t const Size = I->Size();
1513 if (Progress != NULL)
1514 Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1515 CurrentSize += Size;
1516
1517 if (I->Merge(Gen,Progress) == false)
1518 mergeFailure = true;
1519 };
b07aeb1a
DK
1520
1521 if (List != NULL)
1522 {
1523 for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1524 {
3fd89e62 1525 if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
b07aeb1a
DK
1526 {
1527 _error->Warning("Duplicate sources.list entry %s",
1528 (*i)->Describe().c_str());
1529 continue;
1530 }
1531
1532 if ((*i)->Merge(Gen, Progress) == false)
1533 return false;
1534
1535 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
5465192b
DK
1536 if (Indexes != NULL)
1537 std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1538 if (mergeFailure)
1539 return false;
b07aeb1a
DK
1540 }
1541 }
1542
5465192b 1543 if (Start != End)
2d11135a 1544 {
5465192b
DK
1545 Gen.SelectReleaseFile("", "");
1546 std::for_each(Start, End, indexFileMerge);
1547 if (mergeFailure)
b2e465d6 1548 return false;
5465192b 1549 }
45415543 1550
b07aeb1a 1551 if (HasFileDeps == true)
45415543 1552 {
2e5f4e45
DK
1553 if (Progress != NULL)
1554 Progress->Done();
b07aeb1a 1555 TotalSize = ComputeSize(List, Start, End);
45415543 1556 CurrentSize = 0;
b07aeb1a 1557 for (std::vector<pkgIndexFile *>::const_iterator I = Files.begin(); I != Files.end(); ++I)
45415543 1558 {
4ad8619b 1559 map_filesize_t Size = (*I)->Size();
2e5f4e45
DK
1560 if (Progress != NULL)
1561 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
45415543
AL
1562 CurrentSize += Size;
1563 if ((*I)->MergeFileProvides(Gen,Progress) == false)
1564 return false;
1565 }
1566 }
5465192b 1567
b35d2f5f
AL
1568 return true;
1569}
1570 /*}}}*/
5465192b
DK
1571// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
1572// ---------------------------------------------------------------------
1573/* This makes sure that the status cache (the cache that has all
1574 index files from the sources list and all local ones) is ready
1575 to be mmaped. If OutMap is not zero then a MMap object representing
1576 the cache will be stored there. This is pretty much mandetory if you
1577 are using AllowMem. AllowMem lets the function be run as non-root
1578 where it builds the cache 'fast' into a memory buffer. */
1579static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1580{
4ad8619b
DK
1581 map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1582 map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1583 map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
dcdf1ef1
DK
1584 Flags |= MMap::Moveable;
1585 if (_config->FindB("APT::Cache-Fallback", false) == true)
1586 Flags |= MMap::Fallback;
1587 if (CacheF != NULL)
1588 return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1589 else
1590 return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1591}
5465192b
DK
1592static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1593 std::string const &FileName)
1594{
1595 FileFd SCacheF(FileName, FileFd::WriteAtomic);
1596 if (_error->PendingError() == true)
1597 return false;
1598
1599 fchmod(SCacheF.Fd(),0644);
1600
1601 // Write out the main data
1602 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1603 return _error->Error(_("IO Error saving source cache"));
1604 SCacheF.Sync();
1605
1606 // Write out the proper header
1607 Gen->GetCache().HeaderP->Dirty = false;
1608 if (SCacheF.Seek(0) == false ||
1609 SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1610 return _error->Error(_("IO Error saving source cache"));
1611 Gen->GetCache().HeaderP->Dirty = true;
1612 SCacheF.Sync();
1613 return true;
1614}
1615static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1616 SPtr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1617{
1618 Map = CreateDynamicMMap(NULL, 0);
1619 FileFd CacheF(FileName, FileFd::ReadOnly);
1620 map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1621 if ((alloc == 0 && _error->PendingError())
1622 || CacheF.Read((unsigned char *)Map->Data() + alloc,
1623 CacheF.Size()) == false)
1624 return false;
1625 Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1626 return true;
1627}
453b82a3 1628APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
2e5f4e45
DK
1629 MMap **OutMap, bool AllowMem)
1630 { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1631bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
b2e465d6 1632 MMap **OutMap,bool AllowMem)
b35d2f5f 1633{
c8e572e3 1634 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
73688d27
DK
1635
1636 std::vector<pkgIndexFile *> Files;
b2e465d6
AL
1637 if (_system->AddStatusFiles(Files) == false)
1638 return false;
c5f44afc 1639
b2e465d6 1640 // Decide if we can write to the files..
c8e572e3
MV
1641 string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1642 string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1cd1c398
DK
1643
1644 // ensure the cache directory exists
1645 if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1646 {
1647 string dir = _config->FindDir("Dir::Cache");
1648 size_t const len = dir.size();
1649 if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1650 dir = dir.substr(0, len - 5);
1651 if (CacheFile.empty() == false)
1652 CreateDirectory(dir, flNotFile(CacheFile));
1653 if (SrcCacheFile.empty() == false)
1654 CreateDirectory(dir, flNotFile(SrcCacheFile));
1655 }
1656
2e5f4e45
DK
1657 if (Progress != NULL)
1658 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1659
5465192b
DK
1660 bool pkgcache_fine = false;
1661 bool srcpkgcache_fine = false;
1662 bool volatile_fine = List.GetVolatileFiles().empty();
1663
1664 if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true)
b2e465d6 1665 {
c8e572e3 1666 if (Debug == true)
5465192b
DK
1667 std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1668 pkgcache_fine = true;
1669 srcpkgcache_fine = true;
b2e465d6 1670 }
5465192b 1671 if (pkgcache_fine == false)
b2e465d6 1672 {
5465192b 1673 if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
41b4dee4 1674 {
41b4dee4 1675 if (Debug == true)
5465192b
DK
1676 std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1677 srcpkgcache_fine = true;
41b4dee4 1678 }
b35d2f5f 1679 }
5465192b
DK
1680
1681 if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
8ce4327b 1682 {
5465192b
DK
1683 if (Progress != NULL)
1684 Progress->OverallProgress(1,1,1,_("Reading package lists"));
1685 return true;
8ce4327b 1686 }
5465192b
DK
1687
1688 bool Writeable = false;
1689 if (srcpkgcache_fine == false || pkgcache_fine == false)
2d11135a 1690 {
5465192b
DK
1691 if (CacheFile.empty() == false)
1692 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1693 else if (SrcCacheFile.empty() == false)
1694 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1695
c8e572e3 1696 if (Debug == true)
5465192b
DK
1697 std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1698
1699 if (Writeable == false && AllowMem == false)
1700 {
1701 if (CacheFile.empty() == false)
1702 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1703 else if (SrcCacheFile.empty() == false)
1704 return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str());
1705 else
1706 return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!");
1707 }
1708 }
b2e465d6 1709
5465192b
DK
1710 // At this point we know we need to construct something, so get storage ready
1711 SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1712 if (Debug == true)
1713 std::clog << "Open memory Map (not filebased)" << std::endl;
c8e572e3 1714
5465192b
DK
1715 std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1716 map_filesize_t CurrentSize = 0;
1717 std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1718 map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1719 if (srcpkgcache_fine == true && pkgcache_fine == false)
1720 {
1721 if (Debug == true)
1722 std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1723 if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
b2e465d6 1724 return false;
5465192b
DK
1725 srcpkgcache_fine = true;
1726 TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
b2e465d6 1727 }
5465192b 1728 else if (srcpkgcache_fine == false)
2d11135a 1729 {
c8e572e3
MV
1730 if (Debug == true)
1731 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
5465192b
DK
1732 Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1733
1734 TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1735 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1736 Files.end(),Files.end()) == false)
b2e465d6 1737 return false;
5465192b 1738
b2e465d6 1739 if (Writeable == true && SrcCacheFile.empty() == false)
5465192b 1740 if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false)
b2e465d6 1741 return false;
5465192b
DK
1742 }
1743
1744 if (pkgcache_fine == false)
1745 {
1746 if (Debug == true)
1747 std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1748 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1749 Files.begin(), Files.end()) == false)
b2e465d6 1750 return false;
5465192b
DK
1751
1752 if (Writeable == true && CacheFile.empty() == false)
1753 if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false)
1754 return false;
2d11135a 1755 }
5465192b 1756
c8e572e3 1757 if (Debug == true)
5465192b 1758 std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
2d11135a 1759
5465192b 1760 if (volatile_fine == false)
2d11135a 1761 {
5465192b 1762 if (Gen == nullptr)
2d11135a 1763 {
5465192b
DK
1764 if (Debug == true)
1765 std::clog << "Populate new MMap with cachefile contents" << std::endl;
1766 if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1767 return false;
2d11135a 1768 }
5465192b
DK
1769
1770 Files = List.GetVolatileFiles();
1771 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1772 Files.begin(), Files.end()) == false)
1773 return false;
2d11135a 1774 }
5465192b
DK
1775
1776 if (OutMap != nullptr)
1777 *OutMap = Map.UnGuard();
1778
1779 if (Debug == true)
1780 std::clog << "Everything is ready for shipping" << std::endl;
b2e465d6
AL
1781 return true;
1782}
1783 /*}}}*/
2e5f4e45 1784// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
b2e465d6
AL
1785// ---------------------------------------------------------------------
1786/* */
453b82a3 1787APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
2e5f4e45
DK
1788 { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1789bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
b2e465d6 1790{
73688d27 1791 std::vector<pkgIndexFile *> Files;
b2e465d6
AL
1792 if (_system->AddStatusFiles(Files) == false)
1793 return false;
dcdf1ef1 1794
5465192b 1795 SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
4ad8619b
DK
1796 map_filesize_t CurrentSize = 0;
1797 map_filesize_t TotalSize = 0;
b2e465d6 1798
b07aeb1a 1799 TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
b2e465d6
AL
1800
1801 // Build the status cache
2e5f4e45
DK
1802 if (Progress != NULL)
1803 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1804 pkgCacheGenerator Gen(Map.Get(),Progress);
2d11135a 1805 if (_error->PendingError() == true)
b2e465d6 1806 return false;
b07aeb1a
DK
1807 if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1808 Files.begin(), Files.end()) == false)
b2e465d6 1809 return false;
25396fb0 1810
b2e465d6
AL
1811 if (_error->PendingError() == true)
1812 return false;
1813 *OutMap = Map.UnGuard();
2d11135a 1814
b2e465d6 1815 return true;
2d11135a
AL
1816}
1817 /*}}}*/
5f4db009 1818// IsDuplicateDescription /*{{{*/
60683765 1819static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
5f4db009
DK
1820 MD5SumValue const &CurMd5, std::string const &CurLang)
1821{
22f07fc5 1822 // Descriptions in the same link-list have all the same md5
7f5aab82 1823 if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
22f07fc5
DK
1824 return false;
1825 for (; Desc.end() == false; ++Desc)
1826 if (Desc.LanguageCode() == CurLang)
5f4db009
DK
1827 return true;
1828 return false;
1829}
1830 /*}}}*/
5a8e963b 1831// CacheGenerator::FinishCache /*{{{*/
65512241 1832bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
5a8e963b
DK
1833{
1834 return true;
1835}
1836 /*}}}*/
c8a4ce6c 1837
6c55f07a 1838pkgCacheGenerator::ListParser::ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {}
c8a4ce6c 1839pkgCacheGenerator::ListParser::~ListParser() {}