]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
just-in-time creation for (explicit) negative deps
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140 13#include <config.h>
b2e465d6 14
094a497d
AL
15#include <apt-pkg/pkgcachegen.h>
16#include <apt-pkg/error.h>
17#include <apt-pkg/version.h>
b35d2f5f
AL
18#include <apt-pkg/progress.h>
19#include <apt-pkg/sourcelist.h>
20#include <apt-pkg/configuration.h>
cdcc6d34 21#include <apt-pkg/strutl.h>
b2e465d6
AL
22#include <apt-pkg/sptr.h>
23#include <apt-pkg/pkgsystem.h>
aea7f4c8 24#include <apt-pkg/macros.h>
472ff00e
DK
25#include <apt-pkg/metaindex.h>
26#include <apt-pkg/fileutl.h>
453b82a3
DK
27#include <apt-pkg/hashsum_template.h>
28#include <apt-pkg/indexfile.h>
29#include <apt-pkg/md5.h>
30#include <apt-pkg/mmap.h>
31#include <apt-pkg/pkgcache.h>
32#include <apt-pkg/cacheiterators.h>
33
34#include <stddef.h>
35#include <string.h>
36#include <iostream>
37#include <string>
e7b470ee 38#include <vector>
578bfd0a
AL
39#include <sys/stat.h>
40#include <unistd.h>
ea542140
DK
41
42#include <apti18n.h>
578bfd0a 43 /*}}}*/
73688d27 44typedef std::vector<pkgIndexFile *>::iterator FileIterator;
d10cef82 45template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
578bfd0a 46
60683765 47static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
5f4db009
DK
48 MD5SumValue const &CurMd5, std::string const &CurLang);
49
73688d27
DK
50using std::string;
51
578bfd0a
AL
52// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
53// ---------------------------------------------------------------------
25396fb0 54/* We set the dirty flag and make sure that is written to the disk */
b2e465d6 55pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45415543 56 Map(*pMap), Cache(pMap,false), Progress(Prog),
6c55f07a 57 CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0), d(NULL)
578bfd0a
AL
58{
59 if (_error->PendingError() == true)
60 return;
b2e465d6 61
578bfd0a
AL
62 if (Map.Size() == 0)
63 {
b2e465d6
AL
64 // Setup the map interface..
65 Cache.HeaderP = (pkgCache::Header *)Map.Data();
c5f44afc
DK
66 if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
67 return;
68
b2e465d6 69 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
c5f44afc 70
b2e465d6 71 // Starting header
578bfd0a 72 *Cache.HeaderP = pkgCache::Header();
e8a7b0b2
DK
73
74 // make room for the hashtables for packages and groups
32ab4bd0 75 if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
e8a7b0b2
DK
76 return;
77
4ad8619b 78 map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
7a223b93
DK
79 if (unlikely(idxVerSysName == 0))
80 return;
a9fe5928 81 Cache.HeaderP->VerSysName = idxVerSysName;
78a5476f 82 map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
7a223b93 83 if (unlikely(idxArchitecture == 0))
a9fe5928 84 return;
7a223b93
DK
85 Cache.HeaderP->Architecture = idxArchitecture;
86
87 std::vector<std::string> archs = APT::Configuration::getArchitectures();
88 if (archs.size() > 1)
89 {
90 std::vector<std::string>::const_iterator a = archs.begin();
91 std::string list = *a;
92 for (++a; a != archs.end(); ++a)
93 list.append(",").append(*a);
4ad8619b 94 map_stringitem_t const idxArchitectures = WriteStringInMap(list);
7a223b93
DK
95 if (unlikely(idxArchitectures == 0))
96 return;
32ab4bd0 97 Cache.HeaderP->SetArchitectures(idxArchitectures);
7a223b93
DK
98 }
99 else
32ab4bd0 100 Cache.HeaderP->SetArchitectures(idxArchitecture);
7a223b93 101
c5f44afc 102 Cache.ReMap();
578bfd0a 103 }
b2e465d6
AL
104 else
105 {
106 // Map directly from the existing file
107 Cache.ReMap();
108 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
109 if (Cache.VS != _system->VS)
110 {
111 _error->Error(_("Cache has an incompatible versioning system"));
112 return;
e8a7b0b2 113 }
b2e465d6 114 }
e8a7b0b2 115
578bfd0a
AL
116 Cache.HeaderP->Dirty = true;
117 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
118}
119 /*}}}*/
120// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
121// ---------------------------------------------------------------------
122/* We sync the data then unset the dirty flag in two steps so as to
123 advoid a problem during a crash */
124pkgCacheGenerator::~pkgCacheGenerator()
125{
126 if (_error->PendingError() == true)
127 return;
128 if (Map.Sync() == false)
129 return;
130
131 Cache.HeaderP->Dirty = false;
1dfda2ce 132 Cache.HeaderP->CacheFileSize = Map.Size();
578bfd0a
AL
133 Map.Sync(0,sizeof(pkgCache::Header));
134}
135 /*}}}*/
a9fe5928
DK
136void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
137 if (oldMap == newMap)
138 return;
139
64dda04b
MV
140 if (_config->FindB("Debug::pkgCacheGen", false))
141 std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
142
a9fe5928
DK
143 Cache.ReMap(false);
144
cf4ff3b7 145 CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
b07aeb1a 146 CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
a9fe5928 147
7635093c 148 for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
a9fe5928 149 i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
f7a35f2e 150 (*i)->ReMap(oldMap, newMap);
7635093c 151 for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
a9fe5928 152 i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
f7a35f2e 153 (*i)->ReMap(oldMap, newMap);
7635093c 154 for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
a9fe5928 155 i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
f7a35f2e 156 (*i)->ReMap(oldMap, newMap);
7635093c 157 for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
a9fe5928 158 i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
f7a35f2e 159 (*i)->ReMap(oldMap, newMap);
7635093c 160 for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
a9fe5928 161 i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
f7a35f2e 162 (*i)->ReMap(oldMap, newMap);
7635093c 163 for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
a9fe5928 164 i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
f7a35f2e 165 (*i)->ReMap(oldMap, newMap);
7635093c 166 for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
a9fe5928 167 i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
f7a35f2e 168 (*i)->ReMap(oldMap, newMap);
b07aeb1a
DK
169 for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
170 i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
171 (*i)->ReMap(oldMap, newMap);
a9fe5928 172} /*}}}*/
7e58ab0c 173// CacheGenerator::WriteStringInMap /*{{{*/
4ad8619b 174map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
7e58ab0c 175 const unsigned long &Len) {
a9fe5928 176 void const * const oldMap = Map.Data();
4ad8619b 177 map_stringitem_t const index = Map.WriteString(String, Len);
a9fe5928
DK
178 if (index != 0)
179 ReMap(oldMap, Map.Data());
180 return index;
7e58ab0c
DK
181}
182 /*}}}*/
183// CacheGenerator::WriteStringInMap /*{{{*/
4ad8619b 184map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
a9fe5928 185 void const * const oldMap = Map.Data();
4ad8619b 186 map_stringitem_t const index = Map.WriteString(String);
a9fe5928
DK
187 if (index != 0)
188 ReMap(oldMap, Map.Data());
189 return index;
7e58ab0c
DK
190}
191 /*}}}*/
4ad8619b 192map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
a9fe5928 193 void const * const oldMap = Map.Data();
4ad8619b 194 map_pointer_t const index = Map.Allocate(size);
a9fe5928
DK
195 if (index != 0)
196 ReMap(oldMap, Map.Data());
197 return index;
7e58ab0c
DK
198}
199 /*}}}*/
578bfd0a
AL
200// CacheGenerator::MergeList - Merge the package list /*{{{*/
201// ---------------------------------------------------------------------
202/* This provides the generation of the entries in the cache. Each loop
203 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
204bool pkgCacheGenerator::MergeList(ListParser &List,
205 pkgCache::VerIterator *OutVer)
578bfd0a
AL
206{
207 List.Owner = this;
0149949b 208
f9eec0e7 209 unsigned int Counter = 0;
0149949b 210 while (List.Step() == true)
578bfd0a 211 {
5bf15716 212 string const PackageName = List.Package();
65a1e968
AL
213 if (PackageName.empty() == true)
214 return false;
5bf15716 215
99a2ea5a
DK
216 Counter++;
217 if (Counter % 100 == 0 && Progress != 0)
218 Progress->Progress(List.Offset());
219
220 string Arch = List.Architecture();
221 string const Version = List.Version();
222 if (Version.empty() == true && Arch.empty() == true)
223 {
8ec00880 224 // package descriptions
99a2ea5a
DK
225 if (MergeListGroup(List, PackageName) == false)
226 return false;
8ec00880 227 continue;
99a2ea5a
DK
228 }
229
230 if (Arch.empty() == true)
c919ad6e
DK
231 {
232 // use the pseudo arch 'none' for arch-less packages
233 Arch = "none";
234 /* We might built a SingleArchCache here, which we don't want to blow up
235 just for these :none packages to a proper MultiArchCache, so just ensure
236 that we have always a native package structure first for SingleArch */
237 pkgCache::PkgIterator NP;
9abb2283 238 Dynamic<pkgCache::PkgIterator> DynPkg(NP);
c919ad6e
DK
239 if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
240 // TRANSLATOR: The first placeholder is a package name,
241 // the other two should be copied verbatim as they include debug info
242 return _error->Error(_("Error occurred while processing %s (%s%d)"),
243 PackageName.c_str(), "NewPackage", 0);
244 }
99a2ea5a 245
8b32e920 246 // Get a pointer to the package structure
9ddf7030 247 pkgCache::PkgIterator Pkg;
a9fe5928 248 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
28166356 249 if (NewPackage(Pkg, PackageName, Arch) == false)
79e9003c
DK
250 // TRANSLATOR: The first placeholder is a package name,
251 // the other two should be copied verbatim as they include debug info
252 return _error->Error(_("Error occurred while processing %s (%s%d)"),
253 PackageName.c_str(), "NewPackage", 1);
8ce4327b 254
99a2ea5a 255
f55a958f
AL
256 if (Version.empty() == true)
257 {
99a2ea5a
DK
258 if (MergeListPackage(List, Pkg) == false)
259 return false;
260 }
261 else
262 {
263 if (MergeListVersion(List, Pkg, Version, OutVer) == false)
264 return false;
265 }
770c32ec 266
99a2ea5a
DK
267 if (OutVer != 0)
268 {
269 FoundFileDeps |= List.HasFileDeps();
270 return true;
f55a958f 271 }
99a2ea5a
DK
272 }
273
4ad8619b 274 if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
275 return _error->Error(_("Wow, you exceeded the number of package "
276 "names this APT is capable of."));
4ad8619b 277 if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
278 return _error->Error(_("Wow, you exceeded the number of versions "
279 "this APT is capable of."));
4ad8619b 280 if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
281 return _error->Error(_("Wow, you exceeded the number of descriptions "
282 "this APT is capable of."));
4ad8619b 283 if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
284 return _error->Error(_("Wow, you exceeded the number of dependencies "
285 "this APT is capable of."));
286
287 FoundFileDeps |= List.HasFileDeps();
288 return true;
289}
290// CacheGenerator::MergeListGroup /*{{{*/
291bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
292{
293 pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
5f4db009
DK
294 // a group has no data on it's own, only packages have it but these
295 // stanzas like this come from Translation- files to add descriptions,
296 // but without a version we don't need a description for it…
99a2ea5a 297 if (Grp.end() == true)
5f4db009 298 return true;
99a2ea5a
DK
299 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
300
301 pkgCache::PkgIterator Pkg;
302 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
303 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
304 if (MergeListPackage(List, Pkg) == false)
305 return false;
306
307 return true;
308}
309 /*}}}*/
310// CacheGenerator::MergeListPackage /*{{{*/
311bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
312{
313 // we first process the package, then the descriptions
314 // (for deb this package processing is in fact a no-op)
315 pkgCache::VerIterator Ver(Cache);
316 Dynamic<pkgCache::VerIterator> DynVer(Ver);
317 if (List.UsePackage(Pkg, Ver) == false)
79e9003c
DK
318 return _error->Error(_("Error occurred while processing %s (%s%d)"),
319 Pkg.Name(), "UsePackage", 1);
99a2ea5a
DK
320
321 // Find the right version to write the description
322 MD5SumValue CurMd5 = List.Description_md5();
ffe3c68e 323 if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
99359751 324 return true;
ffe3c68e 325 std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
99a2ea5a
DK
326 for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
327 {
a65e22c6 328 pkgCache::DescIterator VerDesc = Ver.DescriptionList();
22f07fc5
DK
329
330 // a version can only have one md5 describing it
a65e22c6 331 if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
22f07fc5 332 continue;
99a2ea5a 333
4ad8619b 334 map_stringitem_t md5idx = VerDesc->md5sum;
ffe3c68e
DK
335 for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
336 {
337 // don't add a new description if we have one for the given
338 // md5 && language
339 if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
340 continue;
a65e22c6 341
ffe3c68e
DK
342 AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
343 }
22f07fc5
DK
344
345 // we can stop here as all "same" versions will share the description
346 break;
99a2ea5a
DK
347 }
348
349 return true;
350}
351 /*}}}*/
352// CacheGenerator::MergeListVersion /*{{{*/
353bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
354 std::string const &Version, pkgCache::VerIterator* &OutVer)
355{
356 pkgCache::VerIterator Ver = Pkg.VersionList();
357 Dynamic<pkgCache::VerIterator> DynVer(Ver);
4ad8619b 358 map_pointer_t *LastVer = &Pkg->VersionList;
99a2ea5a
DK
359 void const * oldMap = Map.Data();
360
68134bda 361 unsigned short const Hash = List.VersionHash();
99a2ea5a
DK
362 if (Ver.end() == false)
363 {
364 /* We know the list is sorted so we use that fact in the search.
365 Insertion of new versions is done with correct sorting */
2246928b 366 int Res = 1;
804de19f 367 for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
578bfd0a 368 {
c24972cb 369 Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
7e2b56a3
DK
370 // Version is higher as current version - insert here
371 if (Res > 0)
578bfd0a 372 break;
7e2b56a3 373 // Versionstrings are equal - is hash also equal?
68134bda 374 if (Res == 0 && List.SameVersion(Hash, Ver) == true)
7e2b56a3
DK
375 break;
376 // proceed with the next till we have either the right
377 // or we found another version (which will be lower)
578bfd0a 378 }
7e2b56a3
DK
379
380 /* We already have a version for this item, record that we saw it */
e426a5ff 381 if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
578bfd0a 382 {
f55a958f 383 if (List.UsePackage(Pkg,Ver) == false)
79e9003c
DK
384 return _error->Error(_("Error occurred while processing %s (%s%d)"),
385 Pkg.Name(), "UsePackage", 2);
f78439bf 386
578bfd0a 387 if (NewFileVer(Ver,List) == false)
79e9003c
DK
388 return _error->Error(_("Error occurred while processing %s (%s%d)"),
389 Pkg.Name(), "NewFileVer", 1);
99a2ea5a 390
ddc1d8d0
AL
391 // Read only a single record and return
392 if (OutVer != 0)
393 {
394 *OutVer = Ver;
395 return true;
396 }
99a2ea5a
DK
397
398 return true;
204fbdcc 399 }
99a2ea5a 400 }
204fbdcc 401
99a2ea5a 402 // Add a new version
4ad8619b 403 map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
99a2ea5a 404 if (verindex == 0 && _error->PendingError())
79e9003c
DK
405 return _error->Error(_("Error occurred while processing %s (%s%d)"),
406 Pkg.Name(), "NewVersion", 1);
a9fe5928 407
99a2ea5a 408 if (oldMap != Map.Data())
4ad8619b 409 LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
99a2ea5a 410 *LastVer = verindex;
a52f938b 411
79e9003c
DK
412 if (unlikely(List.NewVersion(Ver) == false))
413 return _error->Error(_("Error occurred while processing %s (%s%d)"),
414 Pkg.Name(), "NewVersion", 2);
a9fe5928 415
79e9003c
DK
416 if (unlikely(List.UsePackage(Pkg,Ver) == false))
417 return _error->Error(_("Error occurred while processing %s (%s%d)"),
418 Pkg.Name(), "UsePackage", 3);
419
420 if (unlikely(NewFileVer(Ver,List) == false))
421 return _error->Error(_("Error occurred while processing %s (%s%d)"),
422 Pkg.Name(), "NewFileVer", 2);
a52f938b 423
5a8e963b
DK
424 pkgCache::GrpIterator Grp = Pkg.Group();
425 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
426
427 /* If it is the first version of this package we need to add implicit
428 Multi-Arch dependencies to all other package versions in the group now -
429 otherwise we just add them for this new version */
430 if (Pkg.VersionList()->NextVer == 0)
431 {
432 pkgCache::PkgIterator P = Grp.PackageList();
433 Dynamic<pkgCache::PkgIterator> DynP(P);
434 for (; P.end() != true; P = Grp.NextPkg(P))
435 {
436 if (P->ID == Pkg->ID)
437 continue;
438 pkgCache::VerIterator V = P.VersionList();
439 Dynamic<pkgCache::VerIterator> DynV(V);
440 for (; V.end() != true; ++V)
441 if (unlikely(AddImplicitDepends(V, Pkg) == false))
442 return _error->Error(_("Error occurred while processing %s (%s%d)"),
443 Pkg.Name(), "AddImplicitDepends", 1);
444 }
c919ad6e
DK
445 /* :none packages are packages without an architecture. They are forbidden by
446 debian-policy, so usually they will only be in (old) dpkg status files -
447 and dpkg will complain about them - and are pretty rare. We therefore do
448 usually not create conflicts while the parent is created, but only if a :none
449 package (= the target) appears. This creates incorrect dependencies on :none
450 for architecture-specific dependencies on the package we copy from, but we
451 will ignore this bug as architecture-specific dependencies are only allowed
452 in jessie and until then the :none packages should be extinct (hopefully).
453 In other words: This should work long enough to allow graceful removal of
454 these packages, it is not supposed to allow users to keep using them … */
455 if (strcmp(Pkg.Arch(), "none") == 0)
456 {
457 pkgCache::PkgIterator M = Grp.FindPreferredPkg();
458 if (M.end() == false && Pkg != M)
459 {
460 pkgCache::DepIterator D = M.RevDependsList();
461 Dynamic<pkgCache::DepIterator> DynD(D);
462 for (; D.end() == false; ++D)
463 {
464 if ((D->Type != pkgCache::Dep::Conflicts &&
465 D->Type != pkgCache::Dep::DpkgBreaks &&
466 D->Type != pkgCache::Dep::Replaces) ||
467 D.ParentPkg().Group() == Grp)
468 continue;
469
4ad8619b 470 map_pointer_t *OldDepLast = NULL;
c919ad6e 471 pkgCache::VerIterator ConVersion = D.ParentVer();
9abb2283 472 Dynamic<pkgCache::VerIterator> DynV(ConVersion);
c919ad6e 473 // duplicate the Conflicts/Breaks/Replaces for :none arch
9c44383f
DK
474 NewDepends(Pkg, ConVersion, D->Version,
475 D->CompareOp, D->Type, OldDepLast);
c919ad6e
DK
476 }
477 }
478 }
5a8e963b
DK
479 }
480 if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
481 return _error->Error(_("Error occurred while processing %s (%s%d)"),
482 Pkg.Name(), "AddImplicitDepends", 2);
a52f938b 483
99a2ea5a
DK
484 // Read only a single record and return
485 if (OutVer != 0)
486 {
487 *OutVer = Ver;
488 return true;
578bfd0a 489 }
0149949b 490
ffe3c68e 491 /* Record the Description(s) based on their master md5sum */
5f4db009 492 MD5SumValue CurMd5 = List.Description_md5();
ffe3c68e 493 if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
5f4db009 494 return true;
5f4db009
DK
495
496 /* Before we add a new description we first search in the group for
497 a version with a description of the same MD5 - if so we reuse this
498 description group instead of creating our own for this version */
5f4db009
DK
499 for (pkgCache::PkgIterator P = Grp.PackageList();
500 P.end() == false; P = Grp.NextPkg(P))
501 {
502 for (pkgCache::VerIterator V = P.VersionList();
503 V.end() == false; ++V)
504 {
ffe3c68e 505 if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
5f4db009
DK
506 continue;
507 Ver->DescriptionList = V->DescriptionList;
5f4db009
DK
508 }
509 }
510
ffe3c68e 511 // We haven't found reusable descriptions, so add the first description(s)
4ad8619b 512 map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
ffe3c68e
DK
513 std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
514 for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
515 if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
516 return false;
517 return true;
518}
519 /*}}}*/
4ad8619b 520bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
ffe3c68e
DK
521{
522 pkgCache::DescIterator Desc;
99a2ea5a 523 Dynamic<pkgCache::DescIterator> DynDesc(Desc);
99a2ea5a 524
4ad8619b 525 map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
5954d4d2
DK
526 if (unlikely(descindex == 0 && _error->PendingError()))
527 return _error->Error(_("Error occurred while processing %s (%s%d)"),
ffe3c68e
DK
528 Ver.ParentPkg().Name(), "NewDescription", 1);
529
530 md5idx = Desc->md5sum;
531 Desc->ParentPkg = Ver.ParentPkg().Index();
a65e22c6 532
ffe3c68e
DK
533 // we add at the end, so that the start is constant as we need
534 // that to be able to efficiently share these lists
535 pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
536 for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
4ad8619b 537 map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
ffe3c68e 538 *LastNextDesc = descindex;
99a2ea5a 539
a65e22c6 540 if (NewFileDesc(Desc,List) == false)
79e9003c 541 return _error->Error(_("Error occurred while processing %s (%s%d)"),
ffe3c68e 542 Ver.ParentPkg().Name(), "NewFileDesc", 1);
45415543 543
578bfd0a
AL
544 return true;
545}
546 /*}}}*/
99a2ea5a 547 /*}}}*/
45415543
AL
548// CacheGenerator::MergeFileProvides - Merge file provides /*{{{*/
549// ---------------------------------------------------------------------
550/* If we found any file depends while parsing the main list we need to
551 resolve them. Since it is undesired to load the entire list of files
552 into the cache as virtual packages we do a two stage effort. MergeList
553 identifies the file depends and this creates Provdies for them by
554 re-parsing all the indexs. */
555bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
556{
557 List.Owner = this;
558
559 unsigned int Counter = 0;
560 while (List.Step() == true)
561 {
562 string PackageName = List.Package();
563 if (PackageName.empty() == true)
564 return false;
565 string Version = List.Version();
566 if (Version.empty() == true)
567 continue;
568
569 pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
a9fe5928 570 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
45415543 571 if (Pkg.end() == true)
79e9003c
DK
572 return _error->Error(_("Error occurred while processing %s (%s%d)"),
573 PackageName.c_str(), "FindPkg", 1);
45415543
AL
574 Counter++;
575 if (Counter % 100 == 0 && Progress != 0)
576 Progress->Progress(List.Offset());
577
68134bda 578 unsigned short Hash = List.VersionHash();
45415543 579 pkgCache::VerIterator Ver = Pkg.VersionList();
a9fe5928 580 Dynamic<pkgCache::VerIterator> DynVer(Ver);
f7f0d6c7 581 for (; Ver.end() == false; ++Ver)
45415543 582 {
68134bda 583 if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
45415543
AL
584 {
585 if (List.CollectFileProvides(Cache,Ver) == false)
79e9003c
DK
586 return _error->Error(_("Error occurred while processing %s (%s%d)"),
587 PackageName.c_str(), "CollectFileProvides", 1);
45415543
AL
588 break;
589 }
590 }
591
592 if (Ver.end() == true)
593 _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
594 }
595
596 return true;
597}
598 /*}}}*/
5bf15716
DK
599// CacheGenerator::NewGroup - Add a new group /*{{{*/
600// ---------------------------------------------------------------------
601/* This creates a new group structure and adds it to the hash table */
33dd02e3
DK
602bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
603{
604 Grp = Cache.FindGrp(Name);
605 if (Grp.end() == false)
606 return true;
5bf15716 607
33dd02e3 608 // Get a structure
4ad8619b 609 map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
33dd02e3
DK
610 if (unlikely(Group == 0))
611 return false;
5bf15716 612
33dd02e3 613 Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
b13ce62c 614 map_stringitem_t const idxName = StoreString(PKGNAME, Name);
a9fe5928 615 if (unlikely(idxName == 0))
33dd02e3 616 return false;
a9fe5928 617 Grp->Name = idxName;
5bf15716 618
33dd02e3
DK
619 // Insert it into the hash table
620 unsigned long const Hash = Cache.Hash(Name);
32ab4bd0 621 map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
aa0fe657
DK
622 while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
623 insertAt = &(Cache.GrpP + *insertAt)->Next;
624 Grp->Next = *insertAt;
625 *insertAt = Group;
5bf15716 626
52c41485 627 Grp->ID = Cache.HeaderP->GroupCount++;
33dd02e3 628 return true;
5bf15716
DK
629}
630 /*}}}*/
578bfd0a
AL
631// CacheGenerator::NewPackage - Add a new package /*{{{*/
632// ---------------------------------------------------------------------
633/* This creates a new package structure and adds it to the hash table */
5bf15716
DK
634bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
635 const string &Arch) {
636 pkgCache::GrpIterator Grp;
a9fe5928 637 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
5bf15716
DK
638 if (unlikely(NewGroup(Grp, Name) == false))
639 return false;
640
641 Pkg = Grp.FindPkg(Arch);
642 if (Pkg.end() == false)
643 return true;
a52f938b 644
578bfd0a 645 // Get a structure
4ad8619b 646 map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
5bf15716 647 if (unlikely(Package == 0))
578bfd0a 648 return false;
f55a958f 649 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
5bf15716 650
ecc138f8
DK
651 // Set the name, arch and the ID
652 APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
653 Pkg->Group = Grp.Index();
654 // all is mapped to the native architecture
655 map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
656 if (unlikely(idxArch == 0))
657 return false;
658 Pkg->Arch = idxArch;
659 Pkg->ID = Cache.HeaderP->PackageCount++;
660
c408e01e
DK
661 // Insert the package into our package list
662 if (Grp->FirstPackage == 0) // the group is new
663 {
aa0fe657 664 Grp->FirstPackage = Package;
c408e01e 665 // Insert it into the hash table
4ad8619b 666 map_id_t const Hash = Cache.Hash(Name);
32ab4bd0 667 map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
fe86debb 668 while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
32ab4bd0
DK
669 insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
670 Pkg->NextPackage = *insertAt;
aa0fe657 671 *insertAt = Package;
c408e01e
DK
672 }
673 else // Group the Packages together
674 {
ecc138f8
DK
675 // but first get implicit provides done
676 if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
677 {
678 pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
679 if (M.end() == false)
680 for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
681 {
682 if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
683 continue;
684 pkgCache::VerIterator Ver = Prv.OwnerVer();
685 if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
686 ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
687 (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
688 if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
689 return false;
690 }
691
692 for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
693 for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
694 if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
695 if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
696 return false;
697 }
bb0f6a34
DK
698 // and negative dependencies, don't forget negative dependencies
699 {
700 pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
701 if (M.end() == false)
702 for (pkgCache::DepIterator Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
703 {
704 if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
705 continue;
706 if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
707 Dep->Type != pkgCache::Dep::Replaces)
708 continue;
709 pkgCache::VerIterator Ver = Dep.ParentVer();
710 map_pointer_t * unused = NULL;
711 if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
712 return false;
713 }
714 }
ecc138f8 715
c408e01e
DK
716 // this package is the new last package
717 pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
32ab4bd0
DK
718 Pkg->NextPackage = LastPkg->NextPackage;
719 LastPkg->NextPackage = Package;
c408e01e
DK
720 }
721 Grp->LastPackage = Package;
578bfd0a
AL
722 return true;
723}
5a8e963b
DK
724 /*}}}*/
725// CacheGenerator::AddImplicitDepends /*{{{*/
726bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
727 pkgCache::PkgIterator &P,
728 pkgCache::VerIterator &V)
729{
730 // copy P.Arch() into a string here as a cache remap
731 // in NewDepends() later may alter the pointer location
732 string Arch = P.Arch() == NULL ? "" : P.Arch();
4ad8619b 733 map_pointer_t *OldDepLast = NULL;
5a8e963b
DK
734 /* MultiArch handling introduces a lot of implicit Dependencies:
735 - MultiArch: same → Co-Installable if they have the same version
736 - All others conflict with all other group members */
737 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
738 pkgCache::PkgIterator D = G.PackageList();
739 Dynamic<pkgCache::PkgIterator> DynD(D);
4ad8619b 740 map_stringitem_t const VerStrIdx = V->VerStr;
5a8e963b
DK
741 for (; D.end() != true; D = G.NextPkg(D))
742 {
743 if (Arch == D.Arch() || D->VersionList == 0)
744 continue;
745 /* We allow only one installed arch at the time
746 per group, therefore each group member conflicts
747 with all other group members */
748 if (coInstall == true)
749 {
750 // Replaces: ${self}:other ( << ${binary:Version})
9c44383f 751 NewDepends(D, V, VerStrIdx,
8c7af4d4 752 pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
5a8e963b
DK
753 OldDepLast);
754 // Breaks: ${self}:other (!= ${binary:Version})
9c44383f 755 NewDepends(D, V, VerStrIdx,
8c7af4d4 756 pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
5a8e963b
DK
757 OldDepLast);
758 } else {
759 // Conflicts: ${self}:other
9c44383f 760 NewDepends(D, V, 0,
8c7af4d4 761 pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
5a8e963b
DK
762 OldDepLast);
763 }
764 }
765 return true;
766}
767bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
768 pkgCache::PkgIterator &D)
769{
770 /* MultiArch handling introduces a lot of implicit Dependencies:
771 - MultiArch: same → Co-Installable if they have the same version
772 - All others conflict with all other group members */
4ad8619b 773 map_pointer_t *OldDepLast = NULL;
5a8e963b
DK
774 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
775 if (coInstall == true)
776 {
4ad8619b 777 map_stringitem_t const VerStrIdx = V->VerStr;
5a8e963b 778 // Replaces: ${self}:other ( << ${binary:Version})
9c44383f 779 NewDepends(D, V, VerStrIdx,
8c7af4d4 780 pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
5a8e963b
DK
781 OldDepLast);
782 // Breaks: ${self}:other (!= ${binary:Version})
9c44383f 783 NewDepends(D, V, VerStrIdx,
8c7af4d4 784 pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
5a8e963b
DK
785 OldDepLast);
786 } else {
787 // Conflicts: ${self}:other
9c44383f 788 NewDepends(D, V, 0,
8c7af4d4 789 pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
5a8e963b
DK
790 OldDepLast);
791 }
792 return true;
793}
794
578bfd0a
AL
795 /*}}}*/
796// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
797// ---------------------------------------------------------------------
798/* */
f55a958f 799bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
800 ListParser &List)
801{
ddc1d8d0
AL
802 if (CurrentFile == 0)
803 return true;
804
dcb79bae 805 // Get a structure
4ad8619b 806 map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
dcb79bae 807 if (VerFile == 0)
ac2c559b 808 return false;
dcb79bae
AL
809
810 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
811 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
812
813 // Link it to the end of the list
4ad8619b 814 map_pointer_t *Last = &Ver->FileList;
f7f0d6c7 815 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
03e39e59
AL
816 Last = &V->NextFile;
817 VF->NextFile = *Last;
818 *Last = VF.Index();
819
dcb79bae
AL
820 VF->Offset = List.Offset();
821 VF->Size = List.Size();
ad00ae81
AL
822 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
823 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
824 Cache.HeaderP->VerFileCount++;
825
f55a958f 826 return true;
578bfd0a
AL
827}
828 /*}}}*/
829// CacheGenerator::NewVersion - Create a new Version /*{{{*/
830// ---------------------------------------------------------------------
f55a958f 831/* This puts a version structure in the linked list */
4ad8619b 832map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
171c75f1 833 const string &VerStr,
4ad8619b
DK
834 map_pointer_t const ParentPkg,
835 unsigned short const Hash,
836 map_pointer_t const Next)
578bfd0a 837{
f55a958f 838 // Get a structure
4ad8619b 839 map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
f55a958f 840 if (Version == 0)
0149949b 841 return 0;
f55a958f
AL
842
843 // Fill it in
844 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
dfe45e1f 845 //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
f55a958f 846 Ver->NextVer = Next;
885594fc
DK
847 Ver->ParentPkg = ParentPkg;
848 Ver->Hash = Hash;
f55a958f 849 Ver->ID = Cache.HeaderP->VersionCount++;
885594fc
DK
850
851 // try to find the version string in the group for reuse
852 pkgCache::PkgIterator Pkg = Ver.ParentPkg();
853 pkgCache::GrpIterator Grp = Pkg.Group();
854 if (Pkg.end() == false && Grp.end() == false)
855 {
856 for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
857 {
858 if (Pkg == P)
859 continue;
860 for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
861 {
862 int const cmp = strcmp(V.VerStr(), VerStr.c_str());
863 if (cmp == 0)
864 {
865 Ver->VerStr = V->VerStr;
866 return Version;
867 }
868 else if (cmp < 0)
869 break;
870 }
871 }
872 }
873 // haven't found the version string, so create
30b45652 874 map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
a9fe5928 875 if (unlikely(idxVerStr == 0))
0149949b 876 return 0;
a9fe5928 877 Ver->VerStr = idxVerStr;
0149949b 878 return Version;
578bfd0a
AL
879}
880 /*}}}*/
a52f938b
OS
881// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
882// ---------------------------------------------------------------------
883/* */
884bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
885 ListParser &List)
886{
887 if (CurrentFile == 0)
888 return true;
889
890 // Get a structure
4ad8619b 891 map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
a52f938b 892 if (DescFile == 0)
c5f44afc 893 return false;
770c32ec 894
a52f938b
OS
895 pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
896 DF->File = CurrentFile - Cache.PkgFileP;
770c32ec 897
a52f938b 898 // Link it to the end of the list
4ad8619b 899 map_pointer_t *Last = &Desc->FileList;
f7f0d6c7 900 for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
a52f938b 901 Last = &D->NextFile;
770c32ec 902
a52f938b
OS
903 DF->NextFile = *Last;
904 *Last = DF.Index();
905
906 DF->Offset = List.Offset();
907 DF->Size = List.Size();
908 if (Cache.HeaderP->MaxDescFileSize < DF->Size)
909 Cache.HeaderP->MaxDescFileSize = DF->Size;
910 Cache.HeaderP->DescFileCount++;
911
912 return true;
913}
914 /*}}}*/
915// CacheGenerator::NewDescription - Create a new Description /*{{{*/
916// ---------------------------------------------------------------------
917/* This puts a description structure in the linked list */
4ad8619b 918map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
a65e22c6
DK
919 const string &Lang,
920 const MD5SumValue &md5sum,
4ad8619b 921 map_stringitem_t const idxmd5str)
a52f938b
OS
922{
923 // Get a structure
4ad8619b 924 map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
a52f938b
OS
925 if (Description == 0)
926 return 0;
927
928 // Fill it in
929 Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
a52f938b 930 Desc->ID = Cache.HeaderP->DescriptionCount++;
78a5476f 931 map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
a65e22c6 932 if (unlikely(idxlanguage_code == 0))
c5f44afc 933 return 0;
a9fe5928 934 Desc->language_code = idxlanguage_code;
a65e22c6
DK
935
936 if (idxmd5str != 0)
937 Desc->md5sum = idxmd5str;
938 else
939 {
4ad8619b 940 map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
a65e22c6
DK
941 if (unlikely(idxmd5sum == 0))
942 return 0;
943 Desc->md5sum = idxmd5sum;
944 }
a52f938b
OS
945
946 return Description;
947}
948 /*}}}*/
25396fb0 949// CacheGenerator::NewDepends - Create a dependency element /*{{{*/
dcb79bae
AL
950// ---------------------------------------------------------------------
951/* This creates a dependency element in the tree. It is linked to the
952 version and to the package that it is pointing to. */
9c44383f
DK
953bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
954 pkgCache::VerIterator &Ver,
4ad8619b 955 map_pointer_t const Version,
8c7af4d4
DK
956 uint8_t const Op,
957 uint8_t const Type,
4ad8619b 958 map_pointer_t* &OldDepLast)
dcb79bae 959{
a9fe5928 960 void const * const oldMap = Map.Data();
dcb79bae 961 // Get a structure
4ad8619b 962 map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
5bf15716 963 if (unlikely(Dependency == 0))
dcb79bae 964 return false;
9c44383f 965
71c9e95b
DK
966 bool isDuplicate = false;
967 map_pointer_t DependencyData = 0;
b291aa59
DK
968 map_pointer_t PreviousData = 0;
969 if (Pkg->RevDepends != 0)
71c9e95b 970 {
b291aa59
DK
971 pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
972 DependencyData = L->DependencyData;
973 do {
974 pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
975 if (Version > D->Version)
976 break;
977 if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
978 {
979 isDuplicate = true;
980 break;
981 }
982 PreviousData = DependencyData;
983 DependencyData = D->NextData;
984 } while (DependencyData != 0);
71c9e95b
DK
985 }
986
987 if (isDuplicate == false)
988 {
71c9e95b
DK
989 DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
990 if (unlikely(DependencyData == 0))
991 return false;
71c9e95b 992 }
5bf15716 993
71c9e95b
DK
994 pkgCache::Dependency * Link = Cache.DepP + Dependency;
995 Link->ParentVer = Ver.Index();
996 Link->DependencyData = DependencyData;
997 Link->ID = Cache.HeaderP->DependsCount++;
998
999 pkgCache::DepIterator Dep(Cache, Link);
71c9e95b
DK
1000 if (isDuplicate == false)
1001 {
1002 Dep->Type = Type;
1003 Dep->CompareOp = Op;
1004 Dep->Version = Version;
1005 Dep->Package = Pkg.Index();
1006 ++Cache.HeaderP->DependsDataCount;
b291aa59
DK
1007 if (PreviousData != 0)
1008 {
1009 pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1010 Dep->NextData = D->NextData;
1011 D->NextData = DependencyData;
1012 }
1013 else if (Pkg->RevDepends != 0)
1014 {
1015 pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1016 Dep->NextData = D->DependencyData;
1017 }
1018 }
1019
1020 if (isDuplicate == true || PreviousData != 0)
1021 {
1022 pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1023 Link->NextRevDepends = L->NextRevDepends;
1024 L->NextRevDepends = Dependency;
71c9e95b
DK
1025 }
1026 else
1027 {
b291aa59
DK
1028 Link->NextRevDepends = Pkg->RevDepends;
1029 Pkg->RevDepends = Dependency;
71c9e95b 1030 }
25396fb0 1031
b291aa59 1032
25396fb0
DK
1033 // Do we know where to link the Dependency to?
1034 if (OldDepLast == NULL)
c1a22377 1035 {
f9eec0e7 1036 OldDepLast = &Ver->DependsList;
f7f0d6c7 1037 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
f9eec0e7 1038 OldDepLast = &D->NextDepends;
a9fe5928 1039 } else if (oldMap != Map.Data())
4ad8619b 1040 OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
45415543 1041
f9eec0e7 1042 Dep->NextDepends = *OldDepLast;
71c9e95b 1043 *OldDepLast = Dependency;
f9eec0e7 1044 OldDepLast = &Dep->NextDepends;
dcb79bae
AL
1045 return true;
1046}
1047 /*}}}*/
25396fb0
DK
1048// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1049// ---------------------------------------------------------------------
1050/* This creates a Group and the Package to link this dependency to if
1051 needed and handles also the caching of the old endpoint */
32b9a14c 1052bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
25396fb0
DK
1053 const string &PackageName,
1054 const string &Arch,
1055 const string &Version,
8c7af4d4
DK
1056 uint8_t const Op,
1057 uint8_t const Type)
25396fb0
DK
1058{
1059 pkgCache::GrpIterator Grp;
a9fe5928 1060 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
25396fb0
DK
1061 if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1062 return false;
1063
25396fb0
DK
1064 // Is it a file dependency?
1065 if (unlikely(PackageName[0] == '/'))
1066 FoundFileDeps = true;
1067
bb0f6a34
DK
1068 map_stringitem_t idxVersion = 0;
1069 if (Version.empty() == false)
1070 {
1071 int const CmpOp = Op & 0x0F;
1072 // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1073 if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
1074 idxVersion = Ver->VerStr;
1075
1076 if (idxVersion == 0)
1077 {
1078 idxVersion = StoreString(VERSIONNUMBER, Version);
1079 if (unlikely(idxVersion == 0))
1080 return false;
1081 }
25396fb0
DK
1082 }
1083
bb0f6a34
DK
1084 bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1085 Type == pkgCache::Dep::Conflicts ||
1086 Type == pkgCache::Dep::Replaces);
1087
1088 pkgCache::PkgIterator Pkg;
1089 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1090 if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1091 {
1092 // Locate the target package
1093 Pkg = Grp.FindPkg(Arch);
1094 if (Pkg.end() == true) {
1095 if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1096 return false;
1097 }
1098
1099 /* Caching the old end point speeds up generation substantially */
1100 if (OldDepVer != Ver) {
1101 OldDepLast = NULL;
1102 OldDepVer = Ver;
1103 }
1104
1105 return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1106 }
1107 else
1108 {
1109 /* Caching the old end point speeds up generation substantially */
1110 if (OldDepVer != Ver) {
1111 OldDepLast = NULL;
1112 OldDepVer = Ver;
1113 }
1114
1115 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1116 {
1117 if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1118 return false;
1119 }
1120 }
1121 return true;
25396fb0
DK
1122}
1123 /*}}}*/
dcb79bae 1124// ListParser::NewProvides - Create a Provides element /*{{{*/
32b9a14c 1125bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
8c7af4d4 1126 const string &PkgName,
67e0766f 1127 const string &PkgArch,
8c7af4d4
DK
1128 const string &Version,
1129 uint8_t const Flags)
dcb79bae 1130{
ecc138f8 1131 pkgCache const &Cache = Owner->Cache;
8efa2a3b
AL
1132
1133 // We do not add self referencing provides
566046f4 1134 if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
959470da 1135 (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
8efa2a3b 1136 return true;
ecc138f8
DK
1137
1138 // Locate the target package
1139 pkgCache::PkgIterator Pkg;
1140 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1141 if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1142 return false;
1143
1144 map_stringitem_t idxProvideVersion = 0;
1145 if (Version.empty() == false) {
1146 idxProvideVersion = StoreString(VERSIONNUMBER, Version);
1147 if (unlikely(idxProvideVersion == 0))
1148 return false;
1149 }
1150 return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1151}
1152bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1153 pkgCache::PkgIterator &Pkg,
1154 map_pointer_t const ProvideVersion,
1155 uint8_t const Flags)
1156{
dcb79bae 1157 // Get a structure
ecc138f8 1158 map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
5bf15716 1159 if (unlikely(Provides == 0))
dcb79bae 1160 return false;
ecc138f8
DK
1161 ++Cache.HeaderP->ProvidesCount;
1162
dcb79bae
AL
1163 // Fill it in
1164 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1165 Prv->Version = Ver.Index();
ecc138f8 1166 Prv->ProvideVersion = ProvideVersion;
8c7af4d4 1167 Prv->Flags = Flags;
dcb79bae
AL
1168 Prv->NextPkgProv = Ver->ProvidesList;
1169 Ver->ProvidesList = Prv.Index();
ecc138f8 1170
dcb79bae
AL
1171 // Link it to the package
1172 Prv->ParentPkg = Pkg.Index();
1173 Prv->NextProvides = Pkg->ProvidesList;
1174 Pkg->ProvidesList = Prv.Index();
ecc138f8
DK
1175 return true;
1176}
1177 /*}}}*/
1178// ListParser::NewProvidesAllArch - add provides for all architectures /*{{{*/
1179bool pkgCacheGenerator::ListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1180 string const &Version, uint8_t const Flags) {
1181 pkgCache &Cache = Owner->Cache;
1182 pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
1183 if (Grp.end() == true)
1184 return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1185 else
1186 {
1187 map_stringitem_t idxProvideVersion = 0;
1188 if (Version.empty() == false) {
1189 idxProvideVersion = StoreString(VERSIONNUMBER, Version);
1190 if (unlikely(idxProvideVersion == 0))
1191 return false;
1192 }
1193
1194 bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1195 bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1196 pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
1197 for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1198 {
1199 if (isImplicit && OwnerPkg == Pkg)
1200 continue;
1201 if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1202 continue;
1203 if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1204 return false;
1205 }
1206 }
dcb79bae
AL
1207 return true;
1208}
1209 /*}}}*/
68134bda
DK
1210bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1211 pkgCache::VerIterator const &Ver)
1212{
1213 return Hash == Ver->Hash;
1214}
1215 /*}}}*/
b07aeb1a
DK
1216// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to /*{{{*/
1217bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1218 unsigned long Flags)
1219{
1220 if (File.empty() && Site.empty())
1221 {
1222 CurrentRlsFile = NULL;
1223 return true;
1224 }
1225
1226 // Get some space for the structure
1227 map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1228 if (unlikely(idxFile == 0))
1229 return false;
1230 CurrentRlsFile = Cache.RlsFileP + idxFile;
1231
1232 // Fill it in
1233 map_stringitem_t const idxFileName = WriteStringInMap(File);
1234 map_stringitem_t const idxSite = StoreString(MIXED, Site);
1235 if (unlikely(idxFileName == 0 || idxSite == 0))
1236 return false;
1237 CurrentRlsFile->FileName = idxFileName;
1238 CurrentRlsFile->Site = idxSite;
1239 CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1240 CurrentRlsFile->Flags = Flags;
1241 CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1242 RlsFileName = File;
1243 Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1244 Cache.HeaderP->ReleaseFileCount++;
1245
1246 return true;
1247}
1248 /*}}}*/
578bfd0a
AL
1249// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
1250// ---------------------------------------------------------------------
1251/* This is used to select which file is to be associated with all newly
b2e465d6 1252 added versions. The caller is responsible for setting the IMS fields. */
b07aeb1a
DK
1253bool pkgCacheGenerator::SelectFile(std::string const &File,
1254 pkgIndexFile const &Index,
e185d8b3 1255 std::string const &Architecture,
b07aeb1a
DK
1256 std::string const &Component,
1257 unsigned long const Flags)
578bfd0a 1258{
578bfd0a 1259 // Get some space for the structure
4ad8619b 1260 map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
a9fe5928 1261 if (unlikely(idxFile == 0))
578bfd0a 1262 return false;
a9fe5928
DK
1263 CurrentFile = Cache.PkgFileP + idxFile;
1264
578bfd0a 1265 // Fill it in
4ad8619b 1266 map_stringitem_t const idxFileName = WriteStringInMap(File);
b07aeb1a 1267 if (unlikely(idxFileName == 0))
a9fe5928
DK
1268 return false;
1269 CurrentFile->FileName = idxFileName;
578bfd0a 1270 CurrentFile->NextFile = Cache.HeaderP->FileList;
e1b74f61 1271 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
78a5476f 1272 map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
a9fe5928
DK
1273 if (unlikely(idxIndexType == 0))
1274 return false;
1275 CurrentFile->IndexType = idxIndexType;
e185d8b3
DK
1276 if (Architecture.empty())
1277 CurrentFile->Architecture = 0;
1278 else
1279 {
1280 map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1281 if (unlikely(arch == 0))
1282 return false;
1283 CurrentFile->Architecture = arch;
1284 }
b07aeb1a
DK
1285 map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1286 if (unlikely(component == 0))
1287 return false;
1288 CurrentFile->Component = component;
1289 CurrentFile->Flags = Flags;
1290 if (CurrentRlsFile != NULL)
1291 CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1292 else
1293 CurrentFile->Release = 0;
578bfd0a 1294 PkgFileName = File;
ad00ae81 1295 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 1296 Cache.HeaderP->PackageFileCount++;
b2e465d6 1297
ddc1d8d0 1298 if (Progress != 0)
b2e465d6 1299 Progress->SubProgress(Index.Size());
8efa2a3b 1300 return true;
578bfd0a
AL
1301}
1302 /*}}}*/
f55a958f
AL
1303// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
1304// ---------------------------------------------------------------------
1305/* This is used to create handles to strings. Given the same text it
1306 always returns the same number */
78a5476f 1307map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
f55a958f
AL
1308 unsigned int Size)
1309{
78a5476f
DK
1310 std::string const key(S, Size);
1311
1312 std::map<std::string,map_stringitem_t> * strings;
1313 switch(type) {
1314 case MIXED: strings = &strMixed; break;
1315 case PKGNAME: strings = &strPkgNames; break;
30b45652 1316 case VERSIONNUMBER: strings = &strVersions; break;
78a5476f
DK
1317 case SECTION: strings = &strSections; break;
1318 default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
f55a958f 1319 }
a9fe5928 1320
78a5476f
DK
1321 std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1322 if (item != strings->end())
1323 return item->second;
a9fe5928 1324
78a5476f
DK
1325 map_stringitem_t const idxString = WriteStringInMap(S,Size);
1326 strings->insert(std::make_pair(key, idxString));
1327 return idxString;
f55a958f
AL
1328}
1329 /*}}}*/
b2e465d6 1330// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 1331// ---------------------------------------------------------------------
b2e465d6
AL
1332/* This just verifies that each file in the list of index files exists,
1333 has matching attributes with the cache and the cache does not have
1334 any extra files. */
2ec858bc
MV
1335static bool CheckValidity(const string &CacheFile,
1336 pkgSourceList &List,
1337 FileIterator Start,
1338 FileIterator End,
1339 MMap **OutMap = 0)
b35d2f5f 1340{
c8e572e3 1341 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
b2e465d6
AL
1342 // No file, certainly invalid
1343 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
c8e572e3
MV
1344 {
1345 if (Debug == true)
1346 std::clog << "CacheFile doesn't exist" << std::endl;
b35d2f5f 1347 return false;
c8e572e3
MV
1348 }
1349
96d14a91 1350 if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
2ec858bc
MV
1351 {
1352 if (Debug == true)
1353 std::clog << "sources.list is newer than the cache" << std::endl;
1354 return false;
1355 }
1356
b2e465d6 1357 // Map it
b35d2f5f 1358 FileFd CacheF(CacheFile,FileFd::ReadOnly);
eb162ff7 1359 SPtr<MMap> Map = new MMap(CacheF,0);
b35d2f5f 1360 pkgCache Cache(Map);
b2e465d6 1361 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f 1362 {
c8e572e3
MV
1363 if (Debug == true)
1364 std::clog << "Errors are pending or Map is empty()" << std::endl;
b35d2f5f
AL
1365 _error->Discard();
1366 return false;
1367 }
b07aeb1a
DK
1368
1369 SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1370 memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1371 std::vector<pkgIndexFile *> Files;
1372 for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1373 {
1374 if (Debug == true)
1375 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
3fd89e62 1376 pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
b07aeb1a
DK
1377 if (RlsFile.end() == true)
1378 {
1379 if (Debug == true)
1380 std::clog << "FindInCache returned end-Pointer" << std::endl;
1381 return false;
1382 }
1383
1384 RlsVisited[RlsFile->ID] = true;
1385 if (Debug == true)
1386 std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1387
1388 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1389 for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1390 if ((*j)->HasPackages())
1391 Files.push_back (*j);
1392 }
1393 for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1394 if (RlsVisited[I] == false)
1395 {
1396 if (Debug == true)
1397 std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1398 return false;
1399 }
1400
1401 for (; Start != End; ++Start)
1402 Files.push_back(*Start);
1403
b2e465d6
AL
1404 /* Now we check every index file, see if it is in the cache,
1405 verify the IMS data and check that it is on the disk too.. */
1406 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1407 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
b07aeb1a 1408 for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
c8e572e3
MV
1409 {
1410 if (Debug == true)
b07aeb1a
DK
1411 std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1412 if ((*PkgFile)->Exists() == false)
b35d2f5f 1413 {
c8e572e3
MV
1414 if (Debug == true)
1415 std::clog << "file doesn't exist" << std::endl;
b2e465d6 1416 continue;
b35d2f5f 1417 }
b2e465d6
AL
1418
1419 // FindInCache is also expected to do an IMS check.
b07aeb1a 1420 pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
b2e465d6 1421 if (File.end() == true)
c8e572e3
MV
1422 {
1423 if (Debug == true)
1424 std::clog << "FindInCache returned end-Pointer" << std::endl;
b35d2f5f 1425 return false;
c8e572e3 1426 }
a52f938b 1427
b2e465d6 1428 Visited[File->ID] = true;
c8e572e3
MV
1429 if (Debug == true)
1430 std::clog << "with ID " << File->ID << " is valid" << std::endl;
b35d2f5f 1431 }
b07aeb1a 1432
b2e465d6
AL
1433 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1434 if (Visited[I] == false)
c8e572e3
MV
1435 {
1436 if (Debug == true)
b07aeb1a 1437 std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
b2e465d6 1438 return false;
c8e572e3 1439 }
b07aeb1a 1440
b35d2f5f
AL
1441 if (_error->PendingError() == true)
1442 {
c8e572e3
MV
1443 if (Debug == true)
1444 {
1445 std::clog << "Validity failed because of pending errors:" << std::endl;
1446 _error->DumpErrors();
1447 }
b35d2f5f
AL
1448 _error->Discard();
1449 return false;
1450 }
b35d2f5f 1451
b2e465d6
AL
1452 if (OutMap != 0)
1453 *OutMap = Map.UnGuard();
b35d2f5f
AL
1454 return true;
1455}
1456 /*}}}*/
b2e465d6 1457// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 1458// ---------------------------------------------------------------------
b2e465d6
AL
1459/* Size is kind of an abstract notion that is only used for the progress
1460 meter */
b07aeb1a 1461static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
b35d2f5f 1462{
4ad8619b 1463 map_filesize_t TotalSize = 0;
b07aeb1a
DK
1464 if (List != NULL)
1465 {
1466 for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1467 {
1468 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1469 for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1470 if ((*j)->HasPackages() == true)
1471 TotalSize += (*j)->Size();
1472 }
1473 }
1474
ef74268b 1475 for (; Start < End; ++Start)
b35d2f5f 1476 {
b2e465d6 1477 if ((*Start)->HasPackages() == false)
ef74268b 1478 continue;
b2e465d6 1479 TotalSize += (*Start)->Size();
b35d2f5f 1480 }
b2e465d6 1481 return TotalSize;
2d11135a
AL
1482}
1483 /*}}}*/
b2e465d6 1484// BuildCache - Merge the list of index files into the cache /*{{{*/
2d11135a 1485// ---------------------------------------------------------------------
b2e465d6
AL
1486/* */
1487static bool BuildCache(pkgCacheGenerator &Gen,
2e5f4e45 1488 OpProgress *Progress,
4ad8619b 1489 map_filesize_t &CurrentSize,map_filesize_t TotalSize,
b07aeb1a 1490 pkgSourceList const * const List,
e7b470ee 1491 FileIterator Start, FileIterator End)
2d11135a 1492{
b07aeb1a
DK
1493 std::vector<pkgIndexFile *> Files;
1494 bool const HasFileDeps = Gen.HasFileDeps();
1495
1496 if (List != NULL)
1497 {
1498 for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1499 {
3fd89e62 1500 if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
b07aeb1a
DK
1501 {
1502 _error->Warning("Duplicate sources.list entry %s",
1503 (*i)->Describe().c_str());
1504 continue;
1505 }
1506
1507 if ((*i)->Merge(Gen, Progress) == false)
1508 return false;
1509
1510 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1511 for (std::vector<pkgIndexFile *>::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
1512 {
1513 if (HasFileDeps)
1514 Files.push_back(*I);
1515
1516 if ((*I)->HasPackages() == false)
1517 continue;
1518
1519 if ((*I)->Exists() == false)
1520 continue;
1521
1522 if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1523 {
1524 _error->Warning("Duplicate sources.list entry %s",
1525 (*I)->Describe().c_str());
1526 continue;
1527 }
1528
1529 map_filesize_t Size = (*I)->Size();
1530 if (Progress != NULL)
1531 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1532 CurrentSize += Size;
1533
1534 if ((*I)->Merge(Gen,Progress) == false)
1535 return false;
1536 }
1537 }
1538 }
1539
1540 Gen.SelectReleaseFile("", "");
45415543 1541 FileIterator I;
f7f0d6c7 1542 for (I = Start; I != End; ++I)
2d11135a 1543 {
b07aeb1a
DK
1544 if (HasFileDeps)
1545 Files.push_back(*I);
1546
45415543 1547 if ((*I)->HasPackages() == false)
2d11135a
AL
1548 continue;
1549
45415543 1550 if ((*I)->Exists() == false)
2d11135a 1551 continue;
b2e465d6 1552
45415543 1553 if ((*I)->FindInCache(Gen.GetCache()).end() == false)
a77ad7c3
AL
1554 {
1555 _error->Warning("Duplicate sources.list entry %s",
45415543 1556 (*I)->Describe().c_str());
a77ad7c3
AL
1557 continue;
1558 }
1559
4ad8619b 1560 map_filesize_t Size = (*I)->Size();
2e5f4e45
DK
1561 if (Progress != NULL)
1562 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
b2e465d6 1563 CurrentSize += Size;
2d11135a 1564
45415543 1565 if ((*I)->Merge(Gen,Progress) == false)
b2e465d6
AL
1566 return false;
1567 }
45415543 1568
b07aeb1a 1569 if (HasFileDeps == true)
45415543 1570 {
2e5f4e45
DK
1571 if (Progress != NULL)
1572 Progress->Done();
b07aeb1a 1573 TotalSize = ComputeSize(List, Start, End);
45415543 1574 CurrentSize = 0;
b07aeb1a 1575 for (std::vector<pkgIndexFile *>::const_iterator I = Files.begin(); I != Files.end(); ++I)
45415543 1576 {
4ad8619b 1577 map_filesize_t Size = (*I)->Size();
2e5f4e45
DK
1578 if (Progress != NULL)
1579 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
45415543
AL
1580 CurrentSize += Size;
1581 if ((*I)->MergeFileProvides(Gen,Progress) == false)
1582 return false;
1583 }
1584 }
2d11135a 1585
b35d2f5f
AL
1586 return true;
1587}
1588 /*}}}*/
dd13742e 1589// CacheGenerator::CreateDynamicMMap - load an mmap with configuration options /*{{{*/
dcdf1ef1 1590DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
4ad8619b
DK
1591 map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1592 map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1593 map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
dcdf1ef1
DK
1594 Flags |= MMap::Moveable;
1595 if (_config->FindB("APT::Cache-Fallback", false) == true)
1596 Flags |= MMap::Fallback;
1597 if (CacheF != NULL)
1598 return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1599 else
1600 return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1601}
dd13742e 1602 /*}}}*/
2e5f4e45 1603// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
b35d2f5f 1604// ---------------------------------------------------------------------
b2e465d6
AL
1605/* This makes sure that the status cache (the cache that has all
1606 index files from the sources list and all local ones) is ready
1607 to be mmaped. If OutMap is not zero then a MMap object representing
1608 the cache will be stored there. This is pretty much mandetory if you
1609 are using AllowMem. AllowMem lets the function be run as non-root
1610 where it builds the cache 'fast' into a memory buffer. */
453b82a3 1611APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
2e5f4e45
DK
1612 MMap **OutMap, bool AllowMem)
1613 { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1614bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
b2e465d6 1615 MMap **OutMap,bool AllowMem)
b35d2f5f 1616{
c8e572e3 1617 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
73688d27
DK
1618
1619 std::vector<pkgIndexFile *> Files;
b07aeb1a 1620 /*
73688d27 1621 for (std::vector<metaIndex *>::const_iterator i = List.begin();
7db98ffc 1622 i != List.end();
f7f0d6c7 1623 ++i)
7db98ffc 1624 {
73688d27
DK
1625 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1626 for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
7db98ffc 1627 j != Indexes->end();
f7f0d6c7 1628 ++j)
7db98ffc
MZ
1629 Files.push_back (*j);
1630 }
b07aeb1a 1631*/
b2e465d6
AL
1632 if (_system->AddStatusFiles(Files) == false)
1633 return false;
c5f44afc 1634
b2e465d6 1635 // Decide if we can write to the files..
c8e572e3
MV
1636 string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1637 string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1cd1c398
DK
1638
1639 // ensure the cache directory exists
1640 if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1641 {
1642 string dir = _config->FindDir("Dir::Cache");
1643 size_t const len = dir.size();
1644 if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1645 dir = dir.substr(0, len - 5);
1646 if (CacheFile.empty() == false)
1647 CreateDirectory(dir, flNotFile(CacheFile));
1648 if (SrcCacheFile.empty() == false)
1649 CreateDirectory(dir, flNotFile(SrcCacheFile));
1650 }
1651
b2e465d6
AL
1652 // Decide if we can write to the cache
1653 bool Writeable = false;
1654 if (CacheFile.empty() == false)
1655 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1656 else
1657 if (SrcCacheFile.empty() == false)
1658 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
c8e572e3
MV
1659 if (Debug == true)
1660 std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1661
b2e465d6
AL
1662 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1663 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
2e5f4e45
DK
1664
1665 if (Progress != NULL)
1666 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1667
b2e465d6 1668 // Cache is OK, Fin.
2ec858bc 1669 if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
b2e465d6 1670 {
2e5f4e45
DK
1671 if (Progress != NULL)
1672 Progress->OverallProgress(1,1,1,_("Reading package lists"));
c8e572e3
MV
1673 if (Debug == true)
1674 std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
b2e465d6
AL
1675 return true;
1676 }
c8e572e3
MV
1677 else if (Debug == true)
1678 std::clog << "pkgcache.bin is NOT valid" << std::endl;
b2e465d6
AL
1679
1680 /* At this point we know we need to reconstruct the package cache,
1681 begin. */
1682 SPtr<FileFd> CacheF;
1683 SPtr<DynamicMMap> Map;
1684 if (Writeable == true && CacheFile.empty() == false)
1685 {
41b4dee4 1686 _error->PushToStack();
b2e465d6 1687 unlink(CacheFile.c_str());
22041bd2 1688 CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
7a3c2ab0 1689 fchmod(CacheF->Fd(),0644);
dcdf1ef1 1690 Map = CreateDynamicMMap(CacheF, MMap::Public);
b35d2f5f 1691 if (_error->PendingError() == true)
41b4dee4
DK
1692 {
1693 delete CacheF.UnGuard();
1694 delete Map.UnGuard();
1695 if (Debug == true)
1696 std::clog << "Open filebased MMap FAILED" << std::endl;
1697 Writeable = false;
1698 if (AllowMem == false)
1699 {
1700 _error->MergeWithStack();
1701 return false;
1702 }
1703 _error->RevertToStack();
1704 }
0952aee6 1705 else
41b4dee4
DK
1706 {
1707 _error->MergeWithStack();
0952aee6
DH
1708 if (Debug == true)
1709 std::clog << "Open filebased MMap" << std::endl;
41b4dee4 1710 }
b35d2f5f 1711 }
41b4dee4 1712 if (Writeable == false || CacheFile.empty() == true)
8ce4327b 1713 {
b2e465d6 1714 // Just build it in memory..
dcdf1ef1 1715 Map = CreateDynamicMMap(NULL);
c8e572e3
MV
1716 if (Debug == true)
1717 std::clog << "Open memory Map (not filebased)" << std::endl;
8ce4327b 1718 }
b35d2f5f 1719
b2e465d6 1720 // Lets try the source cache.
4ad8619b
DK
1721 map_filesize_t CurrentSize = 0;
1722 map_filesize_t TotalSize = 0;
b07aeb1a
DK
1723 if (CheckValidity(SrcCacheFile, List, Files.end(),
1724 Files.end()) == true)
2d11135a 1725 {
c8e572e3
MV
1726 if (Debug == true)
1727 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
b2e465d6
AL
1728 // Preload the map with the source cache
1729 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
4ad8619b 1730 map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
eb162ff7
DK
1731 if ((alloc == 0 && _error->PendingError())
1732 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1733 SCacheF.Size()) == false)
b2e465d6
AL
1734 return false;
1735
b07aeb1a 1736 TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
c8e572e3 1737
b2e465d6 1738 // Build the status cache
2e5f4e45 1739 pkgCacheGenerator Gen(Map.Get(),Progress);
2d11135a 1740 if (_error->PendingError() == true)
b2e465d6 1741 return false;
b07aeb1a
DK
1742 if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1743 Files.begin(),Files.end()) == false)
b2e465d6
AL
1744 return false;
1745 }
1746 else
2d11135a 1747 {
c8e572e3
MV
1748 if (Debug == true)
1749 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
b07aeb1a 1750 TotalSize = ComputeSize(&List, Files.begin(),Files.end());
2d11135a 1751
b2e465d6 1752 // Build the source cache
2e5f4e45 1753 pkgCacheGenerator Gen(Map.Get(),Progress);
b2e465d6
AL
1754 if (_error->PendingError() == true)
1755 return false;
b07aeb1a
DK
1756 if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
1757 Files.end(),Files.end()) == false)
b2e465d6 1758 return false;
2d11135a 1759
b2e465d6
AL
1760 // Write it back
1761 if (Writeable == true && SrcCacheFile.empty() == false)
2d11135a 1762 {
22041bd2 1763 FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
b2e465d6
AL
1764 if (_error->PendingError() == true)
1765 return false;
7a3c2ab0
AL
1766
1767 fchmod(SCacheF.Fd(),0644);
1768
b2e465d6
AL
1769 // Write out the main data
1770 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1771 return _error->Error(_("IO Error saving source cache"));
1772 SCacheF.Sync();
1773
1774 // Write out the proper header
1775 Gen.GetCache().HeaderP->Dirty = false;
1776 if (SCacheF.Seek(0) == false ||
1777 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1778 return _error->Error(_("IO Error saving source cache"));
b2e465d6 1779 Gen.GetCache().HeaderP->Dirty = true;
7a3c2ab0 1780 SCacheF.Sync();
2d11135a
AL
1781 }
1782
b2e465d6 1783 // Build the status cache
b07aeb1a
DK
1784 if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1785 Files.begin(), Files.end()) == false)
b2e465d6 1786 return false;
2d11135a 1787 }
c8e572e3
MV
1788 if (Debug == true)
1789 std::clog << "Caches are ready for shipping" << std::endl;
2d11135a 1790
b2e465d6
AL
1791 if (_error->PendingError() == true)
1792 return false;
1793 if (OutMap != 0)
2d11135a 1794 {
b2e465d6 1795 if (CacheF != 0)
2d11135a 1796 {
b2e465d6 1797 delete Map.UnGuard();
eb162ff7 1798 *OutMap = new MMap(*CacheF,0);
2d11135a 1799 }
b2e465d6
AL
1800 else
1801 {
1802 *OutMap = Map.UnGuard();
1803 }
2d11135a
AL
1804 }
1805
b2e465d6
AL
1806 return true;
1807}
1808 /*}}}*/
2e5f4e45 1809// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
b2e465d6
AL
1810// ---------------------------------------------------------------------
1811/* */
453b82a3 1812APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
2e5f4e45
DK
1813 { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1814bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
b2e465d6 1815{
73688d27 1816 std::vector<pkgIndexFile *> Files;
b2e465d6
AL
1817 if (_system->AddStatusFiles(Files) == false)
1818 return false;
dcdf1ef1
DK
1819
1820 SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
4ad8619b
DK
1821 map_filesize_t CurrentSize = 0;
1822 map_filesize_t TotalSize = 0;
b2e465d6 1823
b07aeb1a 1824 TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
b2e465d6
AL
1825
1826 // Build the status cache
2e5f4e45
DK
1827 if (Progress != NULL)
1828 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1829 pkgCacheGenerator Gen(Map.Get(),Progress);
2d11135a 1830 if (_error->PendingError() == true)
b2e465d6 1831 return false;
b07aeb1a
DK
1832 if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1833 Files.begin(), Files.end()) == false)
b2e465d6 1834 return false;
25396fb0 1835
b2e465d6
AL
1836 if (_error->PendingError() == true)
1837 return false;
1838 *OutMap = Map.UnGuard();
2d11135a 1839
b2e465d6 1840 return true;
2d11135a
AL
1841}
1842 /*}}}*/
5f4db009 1843// IsDuplicateDescription /*{{{*/
60683765 1844static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
5f4db009
DK
1845 MD5SumValue const &CurMd5, std::string const &CurLang)
1846{
22f07fc5 1847 // Descriptions in the same link-list have all the same md5
7f5aab82 1848 if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
22f07fc5
DK
1849 return false;
1850 for (; Desc.end() == false; ++Desc)
1851 if (Desc.LanguageCode() == CurLang)
5f4db009
DK
1852 return true;
1853 return false;
1854}
1855 /*}}}*/
5a8e963b 1856// CacheGenerator::FinishCache /*{{{*/
65512241 1857bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
5a8e963b
DK
1858{
1859 return true;
1860}
1861 /*}}}*/
c8a4ce6c 1862
6c55f07a 1863pkgCacheGenerator::ListParser::ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {}
c8a4ce6c 1864pkgCacheGenerator::ListParser::~ListParser() {}