]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
deal with partially downloaded changelogs
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140 13#include <config.h>
b2e465d6 14
094a497d
AL
15#include <apt-pkg/pkgcachegen.h>
16#include <apt-pkg/error.h>
17#include <apt-pkg/version.h>
b35d2f5f
AL
18#include <apt-pkg/progress.h>
19#include <apt-pkg/sourcelist.h>
20#include <apt-pkg/configuration.h>
b2e465d6 21#include <apt-pkg/pkgsystem.h>
aea7f4c8 22#include <apt-pkg/macros.h>
472ff00e
DK
23#include <apt-pkg/metaindex.h>
24#include <apt-pkg/fileutl.h>
453b82a3
DK
25#include <apt-pkg/hashsum_template.h>
26#include <apt-pkg/indexfile.h>
27#include <apt-pkg/md5.h>
28#include <apt-pkg/mmap.h>
29#include <apt-pkg/pkgcache.h>
30#include <apt-pkg/cacheiterators.h>
31
32#include <stddef.h>
33#include <string.h>
34#include <iostream>
35#include <string>
e7b470ee 36#include <vector>
5465192b
DK
37#include <memory>
38#include <algorithm>
578bfd0a
AL
39#include <sys/stat.h>
40#include <unistd.h>
ea542140
DK
41
42#include <apti18n.h>
8efd5947
DK
43 /*}}}*/
44template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
73688d27 45typedef std::vector<pkgIndexFile *>::iterator FileIterator;
d10cef82 46template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
578bfd0a 47
60683765 48static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
5f4db009
DK
49 MD5SumValue const &CurMd5, std::string const &CurLang);
50
73688d27 51using std::string;
eff0c22e 52using APT::StringView;
73688d27 53
578bfd0a
AL
54// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
55// ---------------------------------------------------------------------
25396fb0 56/* We set the dirty flag and make sure that is written to the disk */
b2e465d6 57pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45415543 58 Map(*pMap), Cache(pMap,false), Progress(Prog),
1d3eea5c 59 CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
a133f79c
DK
60{
61}
62bool pkgCacheGenerator::Start()
578bfd0a 63{
578bfd0a
AL
64 if (Map.Size() == 0)
65 {
b2e465d6
AL
66 // Setup the map interface..
67 Cache.HeaderP = (pkgCache::Header *)Map.Data();
95278287
DK
68 _error->PushToStack();
69 Map.RawAllocate(sizeof(pkgCache::Header));
70 bool const newError = _error->PendingError();
71 _error->MergeWithStack();
72 if (newError)
a133f79c
DK
73 return false;
74 if (Map.Size() <= 0)
75 return false;
c5f44afc 76
b2e465d6 77 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
c5f44afc 78
b2e465d6 79 // Starting header
578bfd0a 80 *Cache.HeaderP = pkgCache::Header();
e8a7b0b2
DK
81
82 // make room for the hashtables for packages and groups
32ab4bd0 83 if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
a133f79c 84 return false;
e8a7b0b2 85
4ad8619b 86 map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
7a223b93 87 if (unlikely(idxVerSysName == 0))
a133f79c 88 return false;
78a5476f 89 map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
7a223b93 90 if (unlikely(idxArchitecture == 0))
a133f79c
DK
91 return false;
92 map_stringitem_t idxArchitectures;
7a223b93
DK
93
94 std::vector<std::string> archs = APT::Configuration::getArchitectures();
95 if (archs.size() > 1)
96 {
97 std::vector<std::string>::const_iterator a = archs.begin();
98 std::string list = *a;
99 for (++a; a != archs.end(); ++a)
100 list.append(",").append(*a);
a133f79c 101 idxArchitectures = WriteStringInMap(list);
7a223b93 102 if (unlikely(idxArchitectures == 0))
a133f79c 103 return false;
7a223b93
DK
104 }
105 else
a133f79c
DK
106 idxArchitectures = idxArchitecture;
107
108 Cache.HeaderP = (pkgCache::Header *)Map.Data();
109 Cache.HeaderP->VerSysName = idxVerSysName;
110 Cache.HeaderP->Architecture = idxArchitecture;
111 Cache.HeaderP->SetArchitectures(idxArchitectures);
7a223b93 112
25c7a09d
JAK
113 // Calculate the hash for the empty map, so ReMap does not fail
114 Cache.HeaderP->CacheFileSize = Cache.CacheHash();
c5f44afc 115 Cache.ReMap();
578bfd0a 116 }
b2e465d6
AL
117 else
118 {
119 // Map directly from the existing file
120 Cache.ReMap();
121 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
122 if (Cache.VS != _system->VS)
a133f79c 123 return _error->Error(_("Cache has an incompatible versioning system"));
b2e465d6 124 }
e8a7b0b2 125
578bfd0a
AL
126 Cache.HeaderP->Dirty = true;
127 Map.Sync(0,sizeof(pkgCache::Header));
a133f79c 128 return true;
578bfd0a
AL
129}
130 /*}}}*/
131// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
132// ---------------------------------------------------------------------
133/* We sync the data then unset the dirty flag in two steps so as to
134 advoid a problem during a crash */
135pkgCacheGenerator::~pkgCacheGenerator()
136{
6789e01e 137 if (_error->PendingError() == true || Map.validData() == false)
578bfd0a
AL
138 return;
139 if (Map.Sync() == false)
140 return;
141
142 Cache.HeaderP->Dirty = false;
25c7a09d
JAK
143 Cache.HeaderP->CacheFileSize = Cache.CacheHash();
144
145 if (_config->FindB("Debug::pkgCacheGen", false))
146 std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
578bfd0a
AL
147 Map.Sync(0,sizeof(pkgCache::Header));
148}
149 /*}}}*/
19819ac5 150void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
a9fe5928
DK
151 if (oldMap == newMap)
152 return;
153
64dda04b
MV
154 if (_config->FindB("Debug::pkgCacheGen", false))
155 std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
156
a9fe5928
DK
157 Cache.ReMap(false);
158
cf4ff3b7 159 CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
b07aeb1a 160 CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
a9fe5928 161
7635093c 162 for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
a9fe5928 163 i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
f7a35f2e 164 (*i)->ReMap(oldMap, newMap);
7635093c 165 for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
a9fe5928 166 i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
f7a35f2e 167 (*i)->ReMap(oldMap, newMap);
7635093c 168 for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
a9fe5928 169 i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
f7a35f2e 170 (*i)->ReMap(oldMap, newMap);
7635093c 171 for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
a9fe5928 172 i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
f7a35f2e 173 (*i)->ReMap(oldMap, newMap);
7635093c 174 for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
a9fe5928 175 i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
f7a35f2e 176 (*i)->ReMap(oldMap, newMap);
7635093c 177 for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
a9fe5928 178 i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
f7a35f2e 179 (*i)->ReMap(oldMap, newMap);
7635093c 180 for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
a9fe5928 181 i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
f7a35f2e 182 (*i)->ReMap(oldMap, newMap);
b07aeb1a
DK
183 for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
184 i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
185 (*i)->ReMap(oldMap, newMap);
7dd0c2eb
JAK
186 for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
187 // Ignore views outside of the cache.
188 if (ViewP->data() < static_cast<const char*>(oldMap)
189 || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
190 continue;
191 const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
192 *ViewP = StringView(data , ViewP->size());
193 }
a9fe5928 194} /*}}}*/
7e58ab0c 195// CacheGenerator::WriteStringInMap /*{{{*/
4ad8619b 196map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
7e58ab0c 197 const unsigned long &Len) {
19819ac5 198 size_t oldSize = Map.Size();
a9fe5928 199 void const * const oldMap = Map.Data();
4ad8619b 200 map_stringitem_t const index = Map.WriteString(String, Len);
a9fe5928 201 if (index != 0)
19819ac5 202 ReMap(oldMap, Map.Data(), oldSize);
a9fe5928 203 return index;
7e58ab0c
DK
204}
205 /*}}}*/
206// CacheGenerator::WriteStringInMap /*{{{*/
4ad8619b 207map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
19819ac5 208 size_t oldSize = Map.Size();
a9fe5928 209 void const * const oldMap = Map.Data();
4ad8619b 210 map_stringitem_t const index = Map.WriteString(String);
a9fe5928 211 if (index != 0)
19819ac5 212 ReMap(oldMap, Map.Data(), oldSize);
a9fe5928 213 return index;
7e58ab0c
DK
214}
215 /*}}}*/
4ad8619b 216map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
19819ac5 217 size_t oldSize = Map.Size();
a9fe5928 218 void const * const oldMap = Map.Data();
4ad8619b 219 map_pointer_t const index = Map.Allocate(size);
a9fe5928 220 if (index != 0)
19819ac5 221 ReMap(oldMap, Map.Data(), oldSize);
a9fe5928 222 return index;
7e58ab0c
DK
223}
224 /*}}}*/
578bfd0a
AL
225// CacheGenerator::MergeList - Merge the package list /*{{{*/
226// ---------------------------------------------------------------------
227/* This provides the generation of the entries in the cache. Each loop
228 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
229bool pkgCacheGenerator::MergeList(ListParser &List,
230 pkgCache::VerIterator *OutVer)
578bfd0a
AL
231{
232 List.Owner = this;
0149949b 233
f9eec0e7 234 unsigned int Counter = 0;
0149949b 235 while (List.Step() == true)
578bfd0a 236 {
5bf15716 237 string const PackageName = List.Package();
65a1e968
AL
238 if (PackageName.empty() == true)
239 return false;
5bf15716 240
99a2ea5a
DK
241 Counter++;
242 if (Counter % 100 == 0 && Progress != 0)
243 Progress->Progress(List.Offset());
244
8efd5947
DK
245 APT::StringView Arch = List.Architecture();
246 Dynamic<APT::StringView> DynArch(Arch);
247 APT::StringView Version = List.Version();
248 Dynamic<APT::StringView> DynVersion(Version);
99a2ea5a
DK
249 if (Version.empty() == true && Arch.empty() == true)
250 {
8ec00880 251 // package descriptions
99a2ea5a
DK
252 if (MergeListGroup(List, PackageName) == false)
253 return false;
8ec00880 254 continue;
99a2ea5a
DK
255 }
256
8b32e920 257 // Get a pointer to the package structure
9ddf7030 258 pkgCache::PkgIterator Pkg;
a9fe5928 259 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
28166356 260 if (NewPackage(Pkg, PackageName, Arch) == false)
79e9003c
DK
261 // TRANSLATOR: The first placeholder is a package name,
262 // the other two should be copied verbatim as they include debug info
263 return _error->Error(_("Error occurred while processing %s (%s%d)"),
264 PackageName.c_str(), "NewPackage", 1);
8ce4327b 265
99a2ea5a 266
f55a958f
AL
267 if (Version.empty() == true)
268 {
99a2ea5a
DK
269 if (MergeListPackage(List, Pkg) == false)
270 return false;
271 }
272 else
273 {
274 if (MergeListVersion(List, Pkg, Version, OutVer) == false)
275 return false;
276 }
770c32ec 277
99a2ea5a 278 if (OutVer != 0)
99a2ea5a 279 return true;
99a2ea5a
DK
280 }
281
4ad8619b 282 if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
283 return _error->Error(_("Wow, you exceeded the number of package "
284 "names this APT is capable of."));
4ad8619b 285 if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
286 return _error->Error(_("Wow, you exceeded the number of versions "
287 "this APT is capable of."));
4ad8619b 288 if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
289 return _error->Error(_("Wow, you exceeded the number of descriptions "
290 "this APT is capable of."));
4ad8619b 291 if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
292 return _error->Error(_("Wow, you exceeded the number of dependencies "
293 "this APT is capable of."));
294
99a2ea5a
DK
295 return true;
296}
297// CacheGenerator::MergeListGroup /*{{{*/
298bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
299{
300 pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
5f4db009
DK
301 // a group has no data on it's own, only packages have it but these
302 // stanzas like this come from Translation- files to add descriptions,
303 // but without a version we don't need a description for it…
99a2ea5a 304 if (Grp.end() == true)
5f4db009 305 return true;
99a2ea5a
DK
306 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
307
308 pkgCache::PkgIterator Pkg;
309 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
310 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
311 if (MergeListPackage(List, Pkg) == false)
312 return false;
313
314 return true;
315}
316 /*}}}*/
317// CacheGenerator::MergeListPackage /*{{{*/
318bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
319{
320 // we first process the package, then the descriptions
321 // (for deb this package processing is in fact a no-op)
322 pkgCache::VerIterator Ver(Cache);
323 Dynamic<pkgCache::VerIterator> DynVer(Ver);
324 if (List.UsePackage(Pkg, Ver) == false)
79e9003c
DK
325 return _error->Error(_("Error occurred while processing %s (%s%d)"),
326 Pkg.Name(), "UsePackage", 1);
99a2ea5a
DK
327
328 // Find the right version to write the description
329 MD5SumValue CurMd5 = List.Description_md5();
ffe3c68e 330 std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
99a2ea5a
DK
331 for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
332 {
a65e22c6 333 pkgCache::DescIterator VerDesc = Ver.DescriptionList();
22f07fc5
DK
334
335 // a version can only have one md5 describing it
a65e22c6 336 if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
22f07fc5 337 continue;
99a2ea5a 338
4ad8619b 339 map_stringitem_t md5idx = VerDesc->md5sum;
ffe3c68e
DK
340 for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
341 {
342 // don't add a new description if we have one for the given
343 // md5 && language
344 if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
345 continue;
a65e22c6 346
ffe3c68e
DK
347 AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
348 }
22f07fc5
DK
349
350 // we can stop here as all "same" versions will share the description
351 break;
99a2ea5a
DK
352 }
353
354 return true;
355}
356 /*}}}*/
357// CacheGenerator::MergeListVersion /*{{{*/
358bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
8efd5947 359 APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
99a2ea5a
DK
360{
361 pkgCache::VerIterator Ver = Pkg.VersionList();
362 Dynamic<pkgCache::VerIterator> DynVer(Ver);
4ad8619b 363 map_pointer_t *LastVer = &Pkg->VersionList;
99a2ea5a
DK
364 void const * oldMap = Map.Data();
365
68134bda 366 unsigned short const Hash = List.VersionHash();
99a2ea5a
DK
367 if (Ver.end() == false)
368 {
369 /* We know the list is sorted so we use that fact in the search.
370 Insertion of new versions is done with correct sorting */
2246928b 371 int Res = 1;
804de19f 372 for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
578bfd0a 373 {
8efd5947
DK
374 char const * const VerStr = Ver.VerStr();
375 Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
376 VerStr, VerStr + strlen(VerStr));
7e2b56a3
DK
377 // Version is higher as current version - insert here
378 if (Res > 0)
578bfd0a 379 break;
7e2b56a3 380 // Versionstrings are equal - is hash also equal?
68134bda 381 if (Res == 0 && List.SameVersion(Hash, Ver) == true)
7e2b56a3
DK
382 break;
383 // proceed with the next till we have either the right
384 // or we found another version (which will be lower)
578bfd0a 385 }
7e2b56a3
DK
386
387 /* We already have a version for this item, record that we saw it */
e426a5ff 388 if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
578bfd0a 389 {
f55a958f 390 if (List.UsePackage(Pkg,Ver) == false)
79e9003c
DK
391 return _error->Error(_("Error occurred while processing %s (%s%d)"),
392 Pkg.Name(), "UsePackage", 2);
f78439bf 393
578bfd0a 394 if (NewFileVer(Ver,List) == false)
79e9003c
DK
395 return _error->Error(_("Error occurred while processing %s (%s%d)"),
396 Pkg.Name(), "NewFileVer", 1);
99a2ea5a 397
ddc1d8d0
AL
398 // Read only a single record and return
399 if (OutVer != 0)
400 {
401 *OutVer = Ver;
402 return true;
403 }
99a2ea5a
DK
404
405 return true;
204fbdcc 406 }
99a2ea5a 407 }
204fbdcc 408
99a2ea5a 409 // Add a new version
4ad8619b 410 map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
95278287 411 if (unlikely(verindex == 0))
79e9003c
DK
412 return _error->Error(_("Error occurred while processing %s (%s%d)"),
413 Pkg.Name(), "NewVersion", 1);
a9fe5928 414
99a2ea5a 415 if (oldMap != Map.Data())
4ad8619b 416 LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
99a2ea5a 417 *LastVer = verindex;
a52f938b 418
79e9003c
DK
419 if (unlikely(List.NewVersion(Ver) == false))
420 return _error->Error(_("Error occurred while processing %s (%s%d)"),
421 Pkg.Name(), "NewVersion", 2);
a9fe5928 422
79e9003c
DK
423 if (unlikely(List.UsePackage(Pkg,Ver) == false))
424 return _error->Error(_("Error occurred while processing %s (%s%d)"),
425 Pkg.Name(), "UsePackage", 3);
426
427 if (unlikely(NewFileVer(Ver,List) == false))
428 return _error->Error(_("Error occurred while processing %s (%s%d)"),
429 Pkg.Name(), "NewFileVer", 2);
a52f938b 430
5a8e963b
DK
431 pkgCache::GrpIterator Grp = Pkg.Group();
432 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
433
434 /* If it is the first version of this package we need to add implicit
435 Multi-Arch dependencies to all other package versions in the group now -
436 otherwise we just add them for this new version */
437 if (Pkg.VersionList()->NextVer == 0)
438 {
439 pkgCache::PkgIterator P = Grp.PackageList();
440 Dynamic<pkgCache::PkgIterator> DynP(P);
441 for (; P.end() != true; P = Grp.NextPkg(P))
442 {
443 if (P->ID == Pkg->ID)
444 continue;
445 pkgCache::VerIterator V = P.VersionList();
446 Dynamic<pkgCache::VerIterator> DynV(V);
447 for (; V.end() != true; ++V)
448 if (unlikely(AddImplicitDepends(V, Pkg) == false))
449 return _error->Error(_("Error occurred while processing %s (%s%d)"),
450 Pkg.Name(), "AddImplicitDepends", 1);
451 }
452 }
453 if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
454 return _error->Error(_("Error occurred while processing %s (%s%d)"),
455 Pkg.Name(), "AddImplicitDepends", 2);
a52f938b 456
99a2ea5a
DK
457 // Read only a single record and return
458 if (OutVer != 0)
459 {
460 *OutVer = Ver;
461 return true;
578bfd0a 462 }
0149949b 463
ffe3c68e 464 /* Record the Description(s) based on their master md5sum */
5f4db009 465 MD5SumValue CurMd5 = List.Description_md5();
5f4db009
DK
466
467 /* Before we add a new description we first search in the group for
468 a version with a description of the same MD5 - if so we reuse this
469 description group instead of creating our own for this version */
5f4db009
DK
470 for (pkgCache::PkgIterator P = Grp.PackageList();
471 P.end() == false; P = Grp.NextPkg(P))
472 {
473 for (pkgCache::VerIterator V = P.VersionList();
474 V.end() == false; ++V)
475 {
ffe3c68e 476 if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
5f4db009
DK
477 continue;
478 Ver->DescriptionList = V->DescriptionList;
5f4db009
DK
479 }
480 }
481
ffe3c68e 482 // We haven't found reusable descriptions, so add the first description(s)
4ad8619b 483 map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
ffe3c68e
DK
484 std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
485 for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
486 if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
487 return false;
488 return true;
489}
490 /*}}}*/
4ad8619b 491bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
ffe3c68e
DK
492{
493 pkgCache::DescIterator Desc;
99a2ea5a 494 Dynamic<pkgCache::DescIterator> DynDesc(Desc);
99a2ea5a 495
4ad8619b 496 map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
95278287 497 if (unlikely(descindex == 0))
5954d4d2 498 return _error->Error(_("Error occurred while processing %s (%s%d)"),
ffe3c68e
DK
499 Ver.ParentPkg().Name(), "NewDescription", 1);
500
501 md5idx = Desc->md5sum;
502 Desc->ParentPkg = Ver.ParentPkg().Index();
a65e22c6 503
ffe3c68e
DK
504 // we add at the end, so that the start is constant as we need
505 // that to be able to efficiently share these lists
506 pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
507 for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
4ad8619b 508 map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
ffe3c68e 509 *LastNextDesc = descindex;
99a2ea5a 510
a65e22c6 511 if (NewFileDesc(Desc,List) == false)
79e9003c 512 return _error->Error(_("Error occurred while processing %s (%s%d)"),
ffe3c68e 513 Ver.ParentPkg().Name(), "NewFileDesc", 1);
45415543 514
578bfd0a
AL
515 return true;
516}
517 /*}}}*/
99a2ea5a 518 /*}}}*/
5bf15716
DK
519// CacheGenerator::NewGroup - Add a new group /*{{{*/
520// ---------------------------------------------------------------------
521/* This creates a new group structure and adds it to the hash table */
eff0c22e 522bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
33dd02e3 523{
7dd0c2eb 524 Dynamic<StringView> DName(Name);
33dd02e3
DK
525 Grp = Cache.FindGrp(Name);
526 if (Grp.end() == false)
527 return true;
5bf15716 528
33dd02e3 529 // Get a structure
4ad8619b 530 map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
33dd02e3
DK
531 if (unlikely(Group == 0))
532 return false;
5bf15716 533
33dd02e3 534 Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
dd592790 535 map_stringitem_t const idxName = StoreString(PKGNAME, Name);
a9fe5928 536 if (unlikely(idxName == 0))
33dd02e3 537 return false;
a9fe5928 538 Grp->Name = idxName;
5bf15716 539
33dd02e3
DK
540 // Insert it into the hash table
541 unsigned long const Hash = Cache.Hash(Name);
32ab4bd0 542 map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
60d523e2
JAK
543
544 while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
aa0fe657
DK
545 insertAt = &(Cache.GrpP + *insertAt)->Next;
546 Grp->Next = *insertAt;
547 *insertAt = Group;
5bf15716 548
52c41485 549 Grp->ID = Cache.HeaderP->GroupCount++;
33dd02e3 550 return true;
5bf15716
DK
551}
552 /*}}}*/
578bfd0a
AL
553// CacheGenerator::NewPackage - Add a new package /*{{{*/
554// ---------------------------------------------------------------------
555/* This creates a new package structure and adds it to the hash table */
eff0c22e
JAK
556bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
557 StringView Arch) {
5bf15716 558 pkgCache::GrpIterator Grp;
7dd0c2eb
JAK
559 Dynamic<StringView> DName(Name);
560 Dynamic<StringView> DArch(Arch);
a9fe5928 561 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
5bf15716
DK
562 if (unlikely(NewGroup(Grp, Name) == false))
563 return false;
564
565 Pkg = Grp.FindPkg(Arch);
566 if (Pkg.end() == false)
567 return true;
a52f938b 568
578bfd0a 569 // Get a structure
4ad8619b 570 map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
5bf15716 571 if (unlikely(Package == 0))
578bfd0a 572 return false;
f55a958f 573 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
5bf15716 574
ecc138f8
DK
575 // Set the name, arch and the ID
576 APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
577 Pkg->Group = Grp.Index();
578 // all is mapped to the native architecture
dd592790 579 map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
ecc138f8
DK
580 if (unlikely(idxArch == 0))
581 return false;
582 Pkg->Arch = idxArch;
583 Pkg->ID = Cache.HeaderP->PackageCount++;
584
c408e01e
DK
585 // Insert the package into our package list
586 if (Grp->FirstPackage == 0) // the group is new
587 {
aa0fe657 588 Grp->FirstPackage = Package;
c408e01e 589 // Insert it into the hash table
4ad8619b 590 map_id_t const Hash = Cache.Hash(Name);
32ab4bd0 591 map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
eff0c22e 592 while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
32ab4bd0
DK
593 insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
594 Pkg->NextPackage = *insertAt;
aa0fe657 595 *insertAt = Package;
c408e01e
DK
596 }
597 else // Group the Packages together
598 {
62428dbc 599 // if sibling is provided by another package, this one is too
ecc138f8
DK
600 {
601 pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
a6deb7a0
JAK
602 if (M.end() == false) {
603 pkgCache::PrvIterator Prv;
604 Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
605 for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
ecc138f8
DK
606 {
607 if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
608 continue;
609 pkgCache::VerIterator Ver = Prv.OwnerVer();
a6deb7a0 610 Dynamic<pkgCache::VerIterator> DynVer(Ver);
ecc138f8
DK
611 if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
612 ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
613 (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
62428dbc
DK
614 {
615 if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
616 continue;
ecc138f8
DK
617 if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
618 return false;
62428dbc 619 }
ecc138f8 620 }
a6deb7a0 621 }
62428dbc
DK
622 }
623 // let M-A:foreign package siblings provide this package
624 {
a6deb7a0
JAK
625 pkgCache::PkgIterator P;
626 pkgCache::VerIterator Ver;
627 Dynamic<pkgCache::PkgIterator> DynP(P);
628 Dynamic<pkgCache::VerIterator> DynVer(Ver);
ecc138f8 629
a6deb7a0 630 for (P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
62428dbc
DK
631 {
632 if (APT::Configuration::checkArchitecture(P.Arch()) == false)
633 continue;
a6deb7a0 634 for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
ecc138f8
DK
635 if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
636 if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
637 return false;
62428dbc 638 }
ecc138f8 639 }
bb0f6a34
DK
640 // and negative dependencies, don't forget negative dependencies
641 {
642 pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
05c001c8
JAK
643 if (M.end() == false) {
644 pkgCache::DepIterator Dep;
645 Dynamic<pkgCache::DepIterator> DynDep(Dep);
646 for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
bb0f6a34
DK
647 {
648 if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
649 continue;
650 if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
651 Dep->Type != pkgCache::Dep::Replaces)
652 continue;
653 pkgCache::VerIterator Ver = Dep.ParentVer();
05c001c8 654 Dynamic<pkgCache::VerIterator> DynVer(Ver);
bb0f6a34
DK
655 map_pointer_t * unused = NULL;
656 if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
657 return false;
658 }
05c001c8 659 }
bb0f6a34 660 }
ecc138f8 661
c408e01e
DK
662 // this package is the new last package
663 pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
32ab4bd0
DK
664 Pkg->NextPackage = LastPkg->NextPackage;
665 LastPkg->NextPackage = Package;
c408e01e
DK
666 }
667 Grp->LastPackage = Package;
3addaba1
DK
668
669 // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
670 if (Arch == "any")
671 {
074564d4 672 size_t const found = Name.rfind(':');
8ee09b58 673 StringView ArchA = Name.substr(found + 1);
530302ef 674 if (ArchA != "any")
3addaba1 675 {
8ee09b58
JAK
676 // ArchA is used inside the loop which might remap (NameA is not used)
677 Dynamic<StringView> DynArchA(ArchA);
530302ef
DK
678 StringView NameA = Name.substr(0, found);
679 pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
3addaba1 680 Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
530302ef 681 if (PkgA.end())
3addaba1 682 {
530302ef
DK
683 Dynamic<StringView> DynNameA(NameA);
684 if (NewPackage(PkgA, NameA, ArchA) == false)
3addaba1
DK
685 return false;
686 }
530302ef
DK
687 if (unlikely(PkgA.end()))
688 return _error->Fatal("NewPackage was successful for %s:%s,"
689 "but the package doesn't exist anyhow!",
690 NameA.to_string().c_str(), ArchA.to_string().c_str());
691 else
3addaba1 692 {
530302ef
DK
693 pkgCache::PrvIterator Prv = PkgA.ProvidesList();
694 for (; Prv.end() == false; ++Prv)
695 {
696 if (Prv.IsMultiArchImplicit())
697 continue;
698 pkgCache::VerIterator V = Prv.OwnerVer();
699 if (ArchA != V.ParentPkg().Arch())
700 continue;
701 if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
702 return false;
703 }
704 pkgCache::VerIterator V = PkgA.VersionList();
705 Dynamic<pkgCache::VerIterator> DynV(V);
706 for (; V.end() == false; ++V)
707 {
708 if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
709 return false;
710 }
3addaba1
DK
711 }
712 }
713 }
578bfd0a
AL
714 return true;
715}
5a8e963b
DK
716 /*}}}*/
717// CacheGenerator::AddImplicitDepends /*{{{*/
718bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
719 pkgCache::PkgIterator &P,
720 pkgCache::VerIterator &V)
721{
8efd5947
DK
722 APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
723 Dynamic<APT::StringView> DynArch(Arch);
4ad8619b 724 map_pointer_t *OldDepLast = NULL;
5a8e963b
DK
725 /* MultiArch handling introduces a lot of implicit Dependencies:
726 - MultiArch: same → Co-Installable if they have the same version
727 - All others conflict with all other group members */
728 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
729 pkgCache::PkgIterator D = G.PackageList();
730 Dynamic<pkgCache::PkgIterator> DynD(D);
4ad8619b 731 map_stringitem_t const VerStrIdx = V->VerStr;
5a8e963b
DK
732 for (; D.end() != true; D = G.NextPkg(D))
733 {
734 if (Arch == D.Arch() || D->VersionList == 0)
735 continue;
736 /* We allow only one installed arch at the time
737 per group, therefore each group member conflicts
738 with all other group members */
739 if (coInstall == true)
740 {
741 // Replaces: ${self}:other ( << ${binary:Version})
9c44383f 742 NewDepends(D, V, VerStrIdx,
8c7af4d4 743 pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
5a8e963b
DK
744 OldDepLast);
745 // Breaks: ${self}:other (!= ${binary:Version})
9c44383f 746 NewDepends(D, V, VerStrIdx,
8c7af4d4 747 pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
5a8e963b
DK
748 OldDepLast);
749 } else {
750 // Conflicts: ${self}:other
9c44383f 751 NewDepends(D, V, 0,
8c7af4d4 752 pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
5a8e963b
DK
753 OldDepLast);
754 }
755 }
756 return true;
757}
758bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
759 pkgCache::PkgIterator &D)
760{
761 /* MultiArch handling introduces a lot of implicit Dependencies:
762 - MultiArch: same → Co-Installable if they have the same version
763 - All others conflict with all other group members */
4ad8619b 764 map_pointer_t *OldDepLast = NULL;
5a8e963b
DK
765 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
766 if (coInstall == true)
767 {
4ad8619b 768 map_stringitem_t const VerStrIdx = V->VerStr;
5a8e963b 769 // Replaces: ${self}:other ( << ${binary:Version})
9c44383f 770 NewDepends(D, V, VerStrIdx,
8c7af4d4 771 pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
5a8e963b
DK
772 OldDepLast);
773 // Breaks: ${self}:other (!= ${binary:Version})
9c44383f 774 NewDepends(D, V, VerStrIdx,
8c7af4d4 775 pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
5a8e963b
DK
776 OldDepLast);
777 } else {
778 // Conflicts: ${self}:other
9c44383f 779 NewDepends(D, V, 0,
8c7af4d4 780 pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
5a8e963b
DK
781 OldDepLast);
782 }
783 return true;
784}
785
578bfd0a
AL
786 /*}}}*/
787// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
788// ---------------------------------------------------------------------
789/* */
f55a958f 790bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
791 ListParser &List)
792{
ddc1d8d0
AL
793 if (CurrentFile == 0)
794 return true;
795
dcb79bae 796 // Get a structure
4ad8619b 797 map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
dcb79bae 798 if (VerFile == 0)
ac2c559b 799 return false;
dcb79bae
AL
800
801 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
802 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
803
804 // Link it to the end of the list
4ad8619b 805 map_pointer_t *Last = &Ver->FileList;
f7f0d6c7 806 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
03e39e59
AL
807 Last = &V->NextFile;
808 VF->NextFile = *Last;
809 *Last = VF.Index();
810
dcb79bae
AL
811 VF->Offset = List.Offset();
812 VF->Size = List.Size();
ad00ae81
AL
813 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
814 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
815 Cache.HeaderP->VerFileCount++;
816
f55a958f 817 return true;
578bfd0a
AL
818}
819 /*}}}*/
820// CacheGenerator::NewVersion - Create a new Version /*{{{*/
821// ---------------------------------------------------------------------
f55a958f 822/* This puts a version structure in the linked list */
4ad8619b 823map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
8efd5947 824 APT::StringView const &VerStr,
4ad8619b
DK
825 map_pointer_t const ParentPkg,
826 unsigned short const Hash,
827 map_pointer_t const Next)
578bfd0a 828{
f55a958f 829 // Get a structure
4ad8619b 830 map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
f55a958f 831 if (Version == 0)
0149949b 832 return 0;
f55a958f
AL
833
834 // Fill it in
835 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
dfe45e1f 836 //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
f55a958f 837 Ver->NextVer = Next;
885594fc
DK
838 Ver->ParentPkg = ParentPkg;
839 Ver->Hash = Hash;
f55a958f 840 Ver->ID = Cache.HeaderP->VersionCount++;
885594fc
DK
841
842 // try to find the version string in the group for reuse
843 pkgCache::PkgIterator Pkg = Ver.ParentPkg();
844 pkgCache::GrpIterator Grp = Pkg.Group();
845 if (Pkg.end() == false && Grp.end() == false)
846 {
847 for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
848 {
849 if (Pkg == P)
850 continue;
851 for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
852 {
8efd5947
DK
853 int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
854 if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
885594fc
DK
855 {
856 Ver->VerStr = V->VerStr;
857 return Version;
858 }
859 else if (cmp < 0)
860 break;
861 }
862 }
863 }
864 // haven't found the version string, so create
30b45652 865 map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
a9fe5928 866 if (unlikely(idxVerStr == 0))
0149949b 867 return 0;
a9fe5928 868 Ver->VerStr = idxVerStr;
0149949b 869 return Version;
578bfd0a
AL
870}
871 /*}}}*/
a52f938b
OS
872// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
873// ---------------------------------------------------------------------
874/* */
875bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
876 ListParser &List)
877{
878 if (CurrentFile == 0)
879 return true;
880
881 // Get a structure
4ad8619b 882 map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
a52f938b 883 if (DescFile == 0)
c5f44afc 884 return false;
770c32ec 885
a52f938b
OS
886 pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
887 DF->File = CurrentFile - Cache.PkgFileP;
770c32ec 888
a52f938b 889 // Link it to the end of the list
4ad8619b 890 map_pointer_t *Last = &Desc->FileList;
f7f0d6c7 891 for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
a52f938b 892 Last = &D->NextFile;
770c32ec 893
a52f938b
OS
894 DF->NextFile = *Last;
895 *Last = DF.Index();
896
897 DF->Offset = List.Offset();
898 DF->Size = List.Size();
899 if (Cache.HeaderP->MaxDescFileSize < DF->Size)
900 Cache.HeaderP->MaxDescFileSize = DF->Size;
901 Cache.HeaderP->DescFileCount++;
902
903 return true;
904}
905 /*}}}*/
906// CacheGenerator::NewDescription - Create a new Description /*{{{*/
907// ---------------------------------------------------------------------
908/* This puts a description structure in the linked list */
4ad8619b 909map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
a65e22c6
DK
910 const string &Lang,
911 const MD5SumValue &md5sum,
4ad8619b 912 map_stringitem_t const idxmd5str)
a52f938b
OS
913{
914 // Get a structure
4ad8619b 915 map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
a52f938b
OS
916 if (Description == 0)
917 return 0;
918
919 // Fill it in
920 Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
a52f938b 921 Desc->ID = Cache.HeaderP->DescriptionCount++;
78a5476f 922 map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
a65e22c6 923 if (unlikely(idxlanguage_code == 0))
c5f44afc 924 return 0;
a9fe5928 925 Desc->language_code = idxlanguage_code;
a65e22c6
DK
926
927 if (idxmd5str != 0)
928 Desc->md5sum = idxmd5str;
929 else
930 {
4ad8619b 931 map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
a65e22c6
DK
932 if (unlikely(idxmd5sum == 0))
933 return 0;
934 Desc->md5sum = idxmd5sum;
935 }
a52f938b
OS
936
937 return Description;
938}
939 /*}}}*/
25396fb0 940// CacheGenerator::NewDepends - Create a dependency element /*{{{*/
dcb79bae
AL
941// ---------------------------------------------------------------------
942/* This creates a dependency element in the tree. It is linked to the
943 version and to the package that it is pointing to. */
9c44383f
DK
944bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
945 pkgCache::VerIterator &Ver,
4ad8619b 946 map_pointer_t const Version,
8c7af4d4
DK
947 uint8_t const Op,
948 uint8_t const Type,
4ad8619b 949 map_pointer_t* &OldDepLast)
dcb79bae 950{
a9fe5928 951 void const * const oldMap = Map.Data();
dcb79bae 952 // Get a structure
4ad8619b 953 map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
5bf15716 954 if (unlikely(Dependency == 0))
dcb79bae 955 return false;
9c44383f 956
71c9e95b
DK
957 bool isDuplicate = false;
958 map_pointer_t DependencyData = 0;
b291aa59
DK
959 map_pointer_t PreviousData = 0;
960 if (Pkg->RevDepends != 0)
71c9e95b 961 {
b291aa59
DK
962 pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
963 DependencyData = L->DependencyData;
964 do {
965 pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
966 if (Version > D->Version)
967 break;
968 if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
969 {
970 isDuplicate = true;
971 break;
972 }
973 PreviousData = DependencyData;
974 DependencyData = D->NextData;
975 } while (DependencyData != 0);
71c9e95b
DK
976 }
977
978 if (isDuplicate == false)
979 {
71c9e95b
DK
980 DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
981 if (unlikely(DependencyData == 0))
982 return false;
71c9e95b 983 }
5bf15716 984
71c9e95b
DK
985 pkgCache::Dependency * Link = Cache.DepP + Dependency;
986 Link->ParentVer = Ver.Index();
987 Link->DependencyData = DependencyData;
988 Link->ID = Cache.HeaderP->DependsCount++;
989
990 pkgCache::DepIterator Dep(Cache, Link);
71c9e95b
DK
991 if (isDuplicate == false)
992 {
993 Dep->Type = Type;
994 Dep->CompareOp = Op;
995 Dep->Version = Version;
996 Dep->Package = Pkg.Index();
997 ++Cache.HeaderP->DependsDataCount;
b291aa59
DK
998 if (PreviousData != 0)
999 {
1000 pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1001 Dep->NextData = D->NextData;
1002 D->NextData = DependencyData;
1003 }
1004 else if (Pkg->RevDepends != 0)
1005 {
1006 pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1007 Dep->NextData = D->DependencyData;
1008 }
1009 }
1010
1011 if (isDuplicate == true || PreviousData != 0)
1012 {
1013 pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1014 Link->NextRevDepends = L->NextRevDepends;
1015 L->NextRevDepends = Dependency;
71c9e95b
DK
1016 }
1017 else
1018 {
b291aa59
DK
1019 Link->NextRevDepends = Pkg->RevDepends;
1020 Pkg->RevDepends = Dependency;
71c9e95b 1021 }
25396fb0 1022
b291aa59 1023
25396fb0
DK
1024 // Do we know where to link the Dependency to?
1025 if (OldDepLast == NULL)
c1a22377 1026 {
f9eec0e7 1027 OldDepLast = &Ver->DependsList;
f7f0d6c7 1028 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
f9eec0e7 1029 OldDepLast = &D->NextDepends;
a9fe5928 1030 } else if (oldMap != Map.Data())
4ad8619b 1031 OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
45415543 1032
f9eec0e7 1033 Dep->NextDepends = *OldDepLast;
71c9e95b 1034 *OldDepLast = Dependency;
f9eec0e7 1035 OldDepLast = &Dep->NextDepends;
dcb79bae
AL
1036 return true;
1037}
1038 /*}}}*/
25396fb0
DK
1039// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1040// ---------------------------------------------------------------------
1041/* This creates a Group and the Package to link this dependency to if
1042 needed and handles also the caching of the old endpoint */
c9443c01 1043bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
eff0c22e
JAK
1044 StringView PackageName,
1045 StringView Arch,
1046 StringView Version,
8c7af4d4
DK
1047 uint8_t const Op,
1048 uint8_t const Type)
25396fb0
DK
1049{
1050 pkgCache::GrpIterator Grp;
a9fe5928 1051 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
7dd0c2eb
JAK
1052 Dynamic<StringView> DynPackageName(PackageName);
1053 Dynamic<StringView> DynArch(Arch);
1054 Dynamic<StringView> DynVersion(Version);
25396fb0
DK
1055 if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1056 return false;
1057
bb0f6a34
DK
1058 map_stringitem_t idxVersion = 0;
1059 if (Version.empty() == false)
1060 {
1061 int const CmpOp = Op & 0x0F;
1062 // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
57590d3c 1063 if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
bb0f6a34
DK
1064 idxVersion = Ver->VerStr;
1065
1066 if (idxVersion == 0)
1067 {
dd592790 1068 idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
bb0f6a34
DK
1069 if (unlikely(idxVersion == 0))
1070 return false;
1071 }
25396fb0
DK
1072 }
1073
bb0f6a34
DK
1074 bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1075 Type == pkgCache::Dep::Conflicts ||
1076 Type == pkgCache::Dep::Replaces);
1077
1078 pkgCache::PkgIterator Pkg;
1079 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1080 if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1081 {
1082 // Locate the target package
1083 Pkg = Grp.FindPkg(Arch);
1084 if (Pkg.end() == true) {
1085 if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1086 return false;
1087 }
1088
1089 /* Caching the old end point speeds up generation substantially */
1090 if (OldDepVer != Ver) {
1091 OldDepLast = NULL;
1092 OldDepVer = Ver;
1093 }
1094
1095 return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1096 }
1097 else
1098 {
1099 /* Caching the old end point speeds up generation substantially */
1100 if (OldDepVer != Ver) {
1101 OldDepLast = NULL;
1102 OldDepVer = Ver;
1103 }
1104
1105 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1106 {
1107 if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1108 return false;
1109 }
1110 }
1111 return true;
25396fb0
DK
1112}
1113 /*}}}*/
dcb79bae 1114// ListParser::NewProvides - Create a Provides element /*{{{*/
c9443c01 1115bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
eff0c22e
JAK
1116 StringView PkgName,
1117 StringView PkgArch,
1118 StringView Version,
8c7af4d4 1119 uint8_t const Flags)
dcb79bae 1120{
ecc138f8 1121 pkgCache const &Cache = Owner->Cache;
7dd0c2eb
JAK
1122 Dynamic<StringView> DynPkgName(PkgName);
1123 Dynamic<StringView> DynArch(PkgArch);
1124 Dynamic<StringView> DynVersion(Version);
8efa2a3b
AL
1125
1126 // We do not add self referencing provides
566046f4 1127 if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
b755de25
DK
1128 (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1129 (Version.empty() || Version == Ver.VerStr()))
8efa2a3b 1130 return true;
ecc138f8
DK
1131
1132 // Locate the target package
1133 pkgCache::PkgIterator Pkg;
1134 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1135 if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1136 return false;
1137
1138 map_stringitem_t idxProvideVersion = 0;
1139 if (Version.empty() == false) {
dd592790 1140 idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
ecc138f8
DK
1141 if (unlikely(idxProvideVersion == 0))
1142 return false;
1143 }
1144 return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1145}
1146bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1147 pkgCache::PkgIterator &Pkg,
1148 map_pointer_t const ProvideVersion,
1149 uint8_t const Flags)
1150{
dcb79bae 1151 // Get a structure
ecc138f8 1152 map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
5bf15716 1153 if (unlikely(Provides == 0))
dcb79bae 1154 return false;
ecc138f8
DK
1155 ++Cache.HeaderP->ProvidesCount;
1156
dcb79bae
AL
1157 // Fill it in
1158 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1159 Prv->Version = Ver.Index();
ecc138f8 1160 Prv->ProvideVersion = ProvideVersion;
8c7af4d4 1161 Prv->Flags = Flags;
dcb79bae
AL
1162 Prv->NextPkgProv = Ver->ProvidesList;
1163 Ver->ProvidesList = Prv.Index();
ecc138f8 1164
dcb79bae
AL
1165 // Link it to the package
1166 Prv->ParentPkg = Pkg.Index();
1167 Prv->NextProvides = Pkg->ProvidesList;
1168 Pkg->ProvidesList = Prv.Index();
ecc138f8
DK
1169 return true;
1170}
1171 /*}}}*/
1172// ListParser::NewProvidesAllArch - add provides for all architectures /*{{{*/
eff0c22e
JAK
1173bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1174 StringView Version, uint8_t const Flags) {
ecc138f8 1175 pkgCache &Cache = Owner->Cache;
a6deb7a0
JAK
1176 pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1177 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
7dd0c2eb
JAK
1178 Dynamic<StringView> DynPackage(Package);
1179 Dynamic<StringView> DynVersion(Version);
a6deb7a0 1180
ecc138f8
DK
1181 if (Grp.end() == true)
1182 return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1183 else
1184 {
1185 map_stringitem_t idxProvideVersion = 0;
1186 if (Version.empty() == false) {
dd592790 1187 idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
ecc138f8
DK
1188 if (unlikely(idxProvideVersion == 0))
1189 return false;
1190 }
1191
1192 bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1193 bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
a6deb7a0
JAK
1194 pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1195 Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1196 pkgCache::PkgIterator Pkg;
1197 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1198 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
ecc138f8
DK
1199 {
1200 if (isImplicit && OwnerPkg == Pkg)
1201 continue;
1202 if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1203 continue;
1204 if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1205 return false;
1206 }
1207 }
dcb79bae
AL
1208 return true;
1209}
1210 /*}}}*/
c9443c01 1211bool pkgCacheListParser::SameVersion(unsigned short const Hash, /*{{{*/
68134bda
DK
1212 pkgCache::VerIterator const &Ver)
1213{
1214 return Hash == Ver->Hash;
1215}
1216 /*}}}*/
b07aeb1a
DK
1217// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to /*{{{*/
1218bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1219 unsigned long Flags)
1220{
1221 if (File.empty() && Site.empty())
1222 {
1223 CurrentRlsFile = NULL;
1224 return true;
1225 }
1226
1227 // Get some space for the structure
1228 map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1229 if (unlikely(idxFile == 0))
1230 return false;
1231 CurrentRlsFile = Cache.RlsFileP + idxFile;
1232
1233 // Fill it in
1234 map_stringitem_t const idxFileName = WriteStringInMap(File);
1235 map_stringitem_t const idxSite = StoreString(MIXED, Site);
1236 if (unlikely(idxFileName == 0 || idxSite == 0))
1237 return false;
1238 CurrentRlsFile->FileName = idxFileName;
1239 CurrentRlsFile->Site = idxSite;
1240 CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1241 CurrentRlsFile->Flags = Flags;
1242 CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1243 RlsFileName = File;
1244 Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1245 Cache.HeaderP->ReleaseFileCount++;
1246
1247 return true;
1248}
1249 /*}}}*/
578bfd0a
AL
1250// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
1251// ---------------------------------------------------------------------
1252/* This is used to select which file is to be associated with all newly
b2e465d6 1253 added versions. The caller is responsible for setting the IMS fields. */
b07aeb1a
DK
1254bool pkgCacheGenerator::SelectFile(std::string const &File,
1255 pkgIndexFile const &Index,
e185d8b3 1256 std::string const &Architecture,
b07aeb1a
DK
1257 std::string const &Component,
1258 unsigned long const Flags)
578bfd0a 1259{
578bfd0a 1260 // Get some space for the structure
4ad8619b 1261 map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
a9fe5928 1262 if (unlikely(idxFile == 0))
578bfd0a 1263 return false;
a9fe5928
DK
1264 CurrentFile = Cache.PkgFileP + idxFile;
1265
578bfd0a 1266 // Fill it in
4ad8619b 1267 map_stringitem_t const idxFileName = WriteStringInMap(File);
b07aeb1a 1268 if (unlikely(idxFileName == 0))
a9fe5928
DK
1269 return false;
1270 CurrentFile->FileName = idxFileName;
578bfd0a 1271 CurrentFile->NextFile = Cache.HeaderP->FileList;
e1b74f61 1272 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
78a5476f 1273 map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
a9fe5928
DK
1274 if (unlikely(idxIndexType == 0))
1275 return false;
1276 CurrentFile->IndexType = idxIndexType;
e185d8b3
DK
1277 if (Architecture.empty())
1278 CurrentFile->Architecture = 0;
1279 else
1280 {
1281 map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1282 if (unlikely(arch == 0))
1283 return false;
1284 CurrentFile->Architecture = arch;
1285 }
b07aeb1a
DK
1286 map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1287 if (unlikely(component == 0))
1288 return false;
1289 CurrentFile->Component = component;
1290 CurrentFile->Flags = Flags;
1291 if (CurrentRlsFile != NULL)
1292 CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1293 else
1294 CurrentFile->Release = 0;
578bfd0a 1295 PkgFileName = File;
ad00ae81 1296 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 1297 Cache.HeaderP->PackageFileCount++;
b2e465d6 1298
ddc1d8d0 1299 if (Progress != 0)
b2e465d6 1300 Progress->SubProgress(Index.Size());
8efa2a3b 1301 return true;
578bfd0a
AL
1302}
1303 /*}}}*/
f55a958f
AL
1304// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
1305// ---------------------------------------------------------------------
1306/* This is used to create handles to strings. Given the same text it
1307 always returns the same number */
78a5476f 1308map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
f55a958f
AL
1309 unsigned int Size)
1310{
869700d8 1311 auto strings = &strMixed;
78a5476f
DK
1312 switch(type) {
1313 case MIXED: strings = &strMixed; break;
1314 case PKGNAME: strings = &strPkgNames; break;
30b45652 1315 case VERSIONNUMBER: strings = &strVersions; break;
78a5476f 1316 case SECTION: strings = &strSections; break;
869700d8 1317 default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
f55a958f 1318 }
a9fe5928 1319
869700d8 1320 auto const item = strings->find({S, Size, nullptr, 0});
78a5476f 1321 if (item != strings->end())
869700d8 1322 return item->item;
a9fe5928 1323
78a5476f 1324 map_stringitem_t const idxString = WriteStringInMap(S,Size);
869700d8 1325 strings->insert({nullptr, Size, this, idxString});
78a5476f 1326 return idxString;
f55a958f
AL
1327}
1328 /*}}}*/
b2e465d6 1329// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 1330// ---------------------------------------------------------------------
b2e465d6
AL
1331/* This just verifies that each file in the list of index files exists,
1332 has matching attributes with the cache and the cache does not have
1333 any extra files. */
95278287
DK
1334class APT_HIDDEN ScopedErrorRevert {
1335public:
1336 ScopedErrorRevert() { _error->PushToStack(); }
1337 ~ScopedErrorRevert() { _error->RevertToStack(); }
1338};
5465192b 1339static bool CheckValidity(const string &CacheFile,
2ec858bc 1340 pkgSourceList &List,
5465192b
DK
1341 FileIterator const Start,
1342 FileIterator const End,
f1616039
JAK
1343 MMap **OutMap = 0,
1344 pkgCache **OutCache = 0)
b35d2f5f 1345{
95278287 1346 ScopedErrorRevert ser;
c8e572e3 1347 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
b2e465d6
AL
1348 // No file, certainly invalid
1349 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
c8e572e3
MV
1350 {
1351 if (Debug == true)
5465192b 1352 std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
b35d2f5f 1353 return false;
c8e572e3
MV
1354 }
1355
96d14a91 1356 if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
2ec858bc
MV
1357 {
1358 if (Debug == true)
1359 std::clog << "sources.list is newer than the cache" << std::endl;
1360 return false;
1361 }
1362
b2e465d6 1363 // Map it
b35d2f5f 1364 FileFd CacheF(CacheFile,FileFd::ReadOnly);
6c413b18 1365 std::unique_ptr<MMap> Map(new MMap(CacheF,0));
6789e01e
DK
1366 if (unlikely(Map->validData()) == false)
1367 return false;
f1616039
JAK
1368 std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1369 pkgCache &Cache = *CacheP.get();
95278287 1370 if (_error->PendingError() || Map->Size() == 0)
b35d2f5f 1371 {
c8e572e3 1372 if (Debug == true)
5465192b 1373 std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
b35d2f5f
AL
1374 return false;
1375 }
b07aeb1a 1376
98cc7fd2
JAK
1377 std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1378 memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
b07aeb1a
DK
1379 std::vector<pkgIndexFile *> Files;
1380 for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1381 {
1382 if (Debug == true)
1383 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
3fd89e62 1384 pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
b07aeb1a
DK
1385 if (RlsFile.end() == true)
1386 {
1387 if (Debug == true)
1388 std::clog << "FindInCache returned end-Pointer" << std::endl;
1389 return false;
1390 }
1391
1392 RlsVisited[RlsFile->ID] = true;
1393 if (Debug == true)
1394 std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1395
5465192b
DK
1396 std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1397 std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1398 [](pkgIndexFile const * const I) { return I->HasPackages(); });
b07aeb1a
DK
1399 }
1400 for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1401 if (RlsVisited[I] == false)
1402 {
1403 if (Debug == true)
1404 std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1405 return false;
1406 }
1407
5465192b 1408 std::copy(Start, End, std::back_inserter(Files));
b07aeb1a 1409
b2e465d6
AL
1410 /* Now we check every index file, see if it is in the cache,
1411 verify the IMS data and check that it is on the disk too.. */
98cc7fd2
JAK
1412 std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1413 memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
b07aeb1a 1414 for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
c8e572e3
MV
1415 {
1416 if (Debug == true)
b07aeb1a
DK
1417 std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1418 if ((*PkgFile)->Exists() == false)
b35d2f5f 1419 {
c8e572e3
MV
1420 if (Debug == true)
1421 std::clog << "file doesn't exist" << std::endl;
b2e465d6 1422 continue;
b35d2f5f 1423 }
b2e465d6
AL
1424
1425 // FindInCache is also expected to do an IMS check.
b07aeb1a 1426 pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
b2e465d6 1427 if (File.end() == true)
c8e572e3
MV
1428 {
1429 if (Debug == true)
1430 std::clog << "FindInCache returned end-Pointer" << std::endl;
b35d2f5f 1431 return false;
c8e572e3 1432 }
a52f938b 1433
b2e465d6 1434 Visited[File->ID] = true;
c8e572e3
MV
1435 if (Debug == true)
1436 std::clog << "with ID " << File->ID << " is valid" << std::endl;
b35d2f5f 1437 }
b07aeb1a 1438
b2e465d6
AL
1439 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1440 if (Visited[I] == false)
c8e572e3
MV
1441 {
1442 if (Debug == true)
b07aeb1a 1443 std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
b2e465d6 1444 return false;
c8e572e3 1445 }
b07aeb1a 1446
b35d2f5f
AL
1447 if (_error->PendingError() == true)
1448 {
c8e572e3
MV
1449 if (Debug == true)
1450 {
1451 std::clog << "Validity failed because of pending errors:" << std::endl;
95278287 1452 _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
c8e572e3 1453 }
b35d2f5f
AL
1454 return false;
1455 }
95278287 1456
b2e465d6 1457 if (OutMap != 0)
6c413b18 1458 *OutMap = Map.release();
f1616039
JAK
1459 if (OutCache != 0)
1460 *OutCache = CacheP.release();
b35d2f5f
AL
1461 return true;
1462}
1463 /*}}}*/
b2e465d6 1464// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 1465// ---------------------------------------------------------------------
b2e465d6
AL
1466/* Size is kind of an abstract notion that is only used for the progress
1467 meter */
b07aeb1a 1468static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
b35d2f5f 1469{
4ad8619b 1470 map_filesize_t TotalSize = 0;
b07aeb1a
DK
1471 if (List != NULL)
1472 {
1473 for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1474 {
1475 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1476 for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1477 if ((*j)->HasPackages() == true)
1478 TotalSize += (*j)->Size();
1479 }
1480 }
1481
ef74268b 1482 for (; Start < End; ++Start)
b35d2f5f 1483 {
b2e465d6 1484 if ((*Start)->HasPackages() == false)
ef74268b 1485 continue;
b2e465d6 1486 TotalSize += (*Start)->Size();
b35d2f5f 1487 }
b2e465d6 1488 return TotalSize;
2d11135a
AL
1489}
1490 /*}}}*/
b2e465d6 1491// BuildCache - Merge the list of index files into the cache /*{{{*/
b2e465d6 1492static bool BuildCache(pkgCacheGenerator &Gen,
5465192b 1493 OpProgress * const Progress,
4ad8619b 1494 map_filesize_t &CurrentSize,map_filesize_t TotalSize,
b07aeb1a 1495 pkgSourceList const * const List,
5465192b 1496 FileIterator const Start, FileIterator const End)
2d11135a 1497{
5465192b
DK
1498 bool mergeFailure = false;
1499
1500 auto const indexFileMerge = [&](pkgIndexFile * const I) {
5465192b
DK
1501 if (I->HasPackages() == false || mergeFailure)
1502 return;
1503
1504 if (I->Exists() == false)
1505 return;
1506
1507 if (I->FindInCache(Gen.GetCache()).end() == false)
1508 {
1509 _error->Warning("Duplicate sources.list entry %s",
1510 I->Describe().c_str());
1511 return;
1512 }
1513
1514 map_filesize_t const Size = I->Size();
1515 if (Progress != NULL)
1516 Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1517 CurrentSize += Size;
1518
1519 if (I->Merge(Gen,Progress) == false)
1520 mergeFailure = true;
1521 };
b07aeb1a
DK
1522
1523 if (List != NULL)
1524 {
1525 for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1526 {
3fd89e62 1527 if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
b07aeb1a
DK
1528 {
1529 _error->Warning("Duplicate sources.list entry %s",
1530 (*i)->Describe().c_str());
1531 continue;
1532 }
1533
1534 if ((*i)->Merge(Gen, Progress) == false)
1535 return false;
1536
1537 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
5465192b
DK
1538 if (Indexes != NULL)
1539 std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1540 if (mergeFailure)
1541 return false;
b07aeb1a
DK
1542 }
1543 }
1544
5465192b 1545 if (Start != End)
2d11135a 1546 {
5465192b
DK
1547 Gen.SelectReleaseFile("", "");
1548 std::for_each(Start, End, indexFileMerge);
1549 if (mergeFailure)
b2e465d6 1550 return false;
5465192b 1551 }
b35d2f5f
AL
1552 return true;
1553}
1554 /*}}}*/
5465192b
DK
1555// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
1556// ---------------------------------------------------------------------
1557/* This makes sure that the status cache (the cache that has all
1558 index files from the sources list and all local ones) is ready
1559 to be mmaped. If OutMap is not zero then a MMap object representing
1560 the cache will be stored there. This is pretty much mandetory if you
1561 are using AllowMem. AllowMem lets the function be run as non-root
1562 where it builds the cache 'fast' into a memory buffer. */
1563static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1564{
4ad8619b
DK
1565 map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1566 map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1567 map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
dcdf1ef1
DK
1568 Flags |= MMap::Moveable;
1569 if (_config->FindB("APT::Cache-Fallback", false) == true)
1570 Flags |= MMap::Fallback;
1571 if (CacheF != NULL)
1572 return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1573 else
1574 return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1575}
5465192b
DK
1576static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1577 std::string const &FileName)
1578{
1579 FileFd SCacheF(FileName, FileFd::WriteAtomic);
95278287 1580 if (SCacheF.IsOpen() == false || SCacheF.Failed())
5465192b
DK
1581 return false;
1582
1583 fchmod(SCacheF.Fd(),0644);
1584
1585 // Write out the main data
1586 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1587 return _error->Error(_("IO Error saving source cache"));
5465192b
DK
1588
1589 // Write out the proper header
1590 Gen->GetCache().HeaderP->Dirty = false;
25c7a09d 1591 Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
5465192b
DK
1592 if (SCacheF.Seek(0) == false ||
1593 SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1594 return _error->Error(_("IO Error saving source cache"));
1595 Gen->GetCache().HeaderP->Dirty = true;
5465192b
DK
1596 return true;
1597}
1598static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
6c413b18 1599 std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
5465192b 1600{
6c413b18 1601 Map.reset(CreateDynamicMMap(NULL, 0));
6789e01e
DK
1602 if (unlikely(Map->validData()) == false)
1603 return false;
5465192b 1604 FileFd CacheF(FileName, FileFd::ReadOnly);
95278287
DK
1605 if (CacheF.IsOpen() == false || CacheF.Failed())
1606 return false;
1607 _error->PushToStack();
5465192b 1608 map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
95278287
DK
1609 bool const newError = _error->PendingError();
1610 _error->MergeWithStack();
1611 if (alloc == 0 && newError)
1612 return false;
1613 if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
5465192b 1614 return false;
6c413b18 1615 Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
a133f79c 1616 return Gen->Start();
5465192b 1617}
5dd00edb 1618bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
2e5f4e45
DK
1619 MMap **OutMap, bool AllowMem)
1620 { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1621bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
c4171975 1622 MMap **OutMap,bool)
f1616039
JAK
1623{
1624 return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1625}
1626bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1627 MMap **OutMap,pkgCache **OutCache, bool)
b35d2f5f 1628{
c4171975 1629 // FIXME: deprecate the ignored AllowMem parameter
c8e572e3 1630 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
73688d27
DK
1631
1632 std::vector<pkgIndexFile *> Files;
b2e465d6
AL
1633 if (_system->AddStatusFiles(Files) == false)
1634 return false;
c5f44afc 1635
b2e465d6 1636 // Decide if we can write to the files..
c8e572e3
MV
1637 string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1638 string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1cd1c398
DK
1639
1640 // ensure the cache directory exists
1641 if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1642 {
1643 string dir = _config->FindDir("Dir::Cache");
1644 size_t const len = dir.size();
1645 if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1646 dir = dir.substr(0, len - 5);
1647 if (CacheFile.empty() == false)
1648 CreateDirectory(dir, flNotFile(CacheFile));
1649 if (SrcCacheFile.empty() == false)
1650 CreateDirectory(dir, flNotFile(SrcCacheFile));
1651 }
1652
2e5f4e45
DK
1653 if (Progress != NULL)
1654 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1655
5465192b
DK
1656 bool pkgcache_fine = false;
1657 bool srcpkgcache_fine = false;
1658 bool volatile_fine = List.GetVolatileFiles().empty();
1659
f1616039
JAK
1660 if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1661 volatile_fine ? OutCache : NULL) == true)
b2e465d6 1662 {
c8e572e3 1663 if (Debug == true)
5465192b
DK
1664 std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1665 pkgcache_fine = true;
1666 srcpkgcache_fine = true;
b2e465d6 1667 }
5465192b 1668 if (pkgcache_fine == false)
b2e465d6 1669 {
5465192b 1670 if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
41b4dee4 1671 {
41b4dee4 1672 if (Debug == true)
5465192b
DK
1673 std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1674 srcpkgcache_fine = true;
41b4dee4 1675 }
b35d2f5f 1676 }
5465192b
DK
1677
1678 if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
8ce4327b 1679 {
5465192b
DK
1680 if (Progress != NULL)
1681 Progress->OverallProgress(1,1,1,_("Reading package lists"));
1682 return true;
8ce4327b 1683 }
5465192b
DK
1684
1685 bool Writeable = false;
1686 if (srcpkgcache_fine == false || pkgcache_fine == false)
2d11135a 1687 {
5465192b
DK
1688 if (CacheFile.empty() == false)
1689 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1690 else if (SrcCacheFile.empty() == false)
1691 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1692
c8e572e3 1693 if (Debug == true)
5465192b 1694 std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
5465192b 1695 }
b2e465d6 1696
5465192b 1697 // At this point we know we need to construct something, so get storage ready
6c413b18 1698 std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
6789e01e
DK
1699 if (unlikely(Map->validData()) == false)
1700 return false;
5465192b
DK
1701 if (Debug == true)
1702 std::clog << "Open memory Map (not filebased)" << std::endl;
c8e572e3 1703
5465192b
DK
1704 std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1705 map_filesize_t CurrentSize = 0;
1706 std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1707 map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1708 if (srcpkgcache_fine == true && pkgcache_fine == false)
1709 {
1710 if (Debug == true)
1711 std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1712 if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
b2e465d6 1713 return false;
5465192b
DK
1714 srcpkgcache_fine = true;
1715 TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
b2e465d6 1716 }
5465192b 1717 else if (srcpkgcache_fine == false)
2d11135a 1718 {
c8e572e3
MV
1719 if (Debug == true)
1720 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
6c413b18 1721 Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
a133f79c
DK
1722 if (Gen->Start() == false)
1723 return false;
5465192b
DK
1724
1725 TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1726 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1727 Files.end(),Files.end()) == false)
b2e465d6 1728 return false;
5465192b 1729
b2e465d6 1730 if (Writeable == true && SrcCacheFile.empty() == false)
6c413b18 1731 if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
b2e465d6 1732 return false;
5465192b
DK
1733 }
1734
1735 if (pkgcache_fine == false)
1736 {
1737 if (Debug == true)
1738 std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1739 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1740 Files.begin(), Files.end()) == false)
b2e465d6 1741 return false;
5465192b
DK
1742
1743 if (Writeable == true && CacheFile.empty() == false)
6c413b18 1744 if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
5465192b 1745 return false;
2d11135a 1746 }
5465192b 1747
c8e572e3 1748 if (Debug == true)
a249b3e6 1749 std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
2d11135a 1750
5465192b 1751 if (volatile_fine == false)
2d11135a 1752 {
5465192b 1753 if (Gen == nullptr)
2d11135a 1754 {
5465192b
DK
1755 if (Debug == true)
1756 std::clog << "Populate new MMap with cachefile contents" << std::endl;
1757 if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1758 return false;
2d11135a 1759 }
5465192b
DK
1760
1761 Files = List.GetVolatileFiles();
1762 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1763 Files.begin(), Files.end()) == false)
1764 return false;
2d11135a 1765 }
5465192b
DK
1766
1767 if (OutMap != nullptr)
6c413b18 1768 *OutMap = Map.release();
5465192b
DK
1769
1770 if (Debug == true)
1771 std::clog << "Everything is ready for shipping" << std::endl;
b2e465d6
AL
1772 return true;
1773}
1774 /*}}}*/
2e5f4e45 1775// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
95278287
DK
1776class APT_HIDDEN ScopedErrorMerge {
1777public:
1778 ScopedErrorMerge() { _error->PushToStack(); }
1779 ~ScopedErrorMerge() { _error->MergeWithStack(); }
1780};
5dd00edb 1781bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
2e5f4e45
DK
1782 { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1783bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
b2e465d6 1784{
73688d27 1785 std::vector<pkgIndexFile *> Files;
b2e465d6
AL
1786 if (_system->AddStatusFiles(Files) == false)
1787 return false;
dcdf1ef1 1788
95278287 1789 ScopedErrorMerge sem;
6c413b18 1790 std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
6789e01e
DK
1791 if (unlikely(Map->validData()) == false)
1792 return false;
4ad8619b
DK
1793 map_filesize_t CurrentSize = 0;
1794 map_filesize_t TotalSize = 0;
b07aeb1a 1795 TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
95278287 1796
b2e465d6 1797 // Build the status cache
2e5f4e45
DK
1798 if (Progress != NULL)
1799 Progress->OverallProgress(0,1,1,_("Reading package lists"));
6c413b18 1800 pkgCacheGenerator Gen(Map.get(),Progress);
a133f79c 1801 if (Gen.Start() == false || _error->PendingError() == true)
b2e465d6 1802 return false;
b07aeb1a
DK
1803 if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1804 Files.begin(), Files.end()) == false)
b2e465d6 1805 return false;
25396fb0 1806
b2e465d6
AL
1807 if (_error->PendingError() == true)
1808 return false;
6c413b18 1809 *OutMap = Map.release();
2d11135a 1810
b2e465d6 1811 return true;
2d11135a
AL
1812}
1813 /*}}}*/
5f4db009 1814// IsDuplicateDescription /*{{{*/
60683765 1815static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
5f4db009
DK
1816 MD5SumValue const &CurMd5, std::string const &CurLang)
1817{
22f07fc5 1818 // Descriptions in the same link-list have all the same md5
7f5aab82 1819 if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
22f07fc5
DK
1820 return false;
1821 for (; Desc.end() == false; ++Desc)
1822 if (Desc.LanguageCode() == CurLang)
5f4db009
DK
1823 return true;
1824 return false;
1825}
1826 /*}}}*/
c8a4ce6c 1827
1d3eea5c 1828pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
c9443c01 1829pkgCacheListParser::~pkgCacheListParser() {}