]> git.saurik.com Git - apt.git/blame - apt-pkg/pkgcachegen.cc
test: Avoid use of /proc/self/fd
[apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140 13#include <config.h>
b2e465d6 14
094a497d
AL
15#include <apt-pkg/pkgcachegen.h>
16#include <apt-pkg/error.h>
17#include <apt-pkg/version.h>
b35d2f5f
AL
18#include <apt-pkg/progress.h>
19#include <apt-pkg/sourcelist.h>
20#include <apt-pkg/configuration.h>
b2e465d6 21#include <apt-pkg/pkgsystem.h>
aea7f4c8 22#include <apt-pkg/macros.h>
472ff00e
DK
23#include <apt-pkg/metaindex.h>
24#include <apt-pkg/fileutl.h>
453b82a3
DK
25#include <apt-pkg/hashsum_template.h>
26#include <apt-pkg/indexfile.h>
27#include <apt-pkg/md5.h>
28#include <apt-pkg/mmap.h>
29#include <apt-pkg/pkgcache.h>
30#include <apt-pkg/cacheiterators.h>
31
32#include <stddef.h>
33#include <string.h>
34#include <iostream>
35#include <string>
e7b470ee 36#include <vector>
5465192b
DK
37#include <memory>
38#include <algorithm>
578bfd0a
AL
39#include <sys/stat.h>
40#include <unistd.h>
ea542140
DK
41
42#include <apti18n.h>
8efd5947
DK
43 /*}}}*/
44template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
73688d27 45typedef std::vector<pkgIndexFile *>::iterator FileIterator;
d10cef82 46template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
578bfd0a 47
60683765 48static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
5f4db009
DK
49 MD5SumValue const &CurMd5, std::string const &CurLang);
50
73688d27 51using std::string;
eff0c22e 52using APT::StringView;
73688d27 53
578bfd0a
AL
54// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
55// ---------------------------------------------------------------------
25396fb0 56/* We set the dirty flag and make sure that is written to the disk */
b2e465d6 57pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45415543 58 Map(*pMap), Cache(pMap,false), Progress(Prog),
1d3eea5c 59 CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
a133f79c
DK
60{
61}
62bool pkgCacheGenerator::Start()
578bfd0a 63{
578bfd0a
AL
64 if (Map.Size() == 0)
65 {
b2e465d6
AL
66 // Setup the map interface..
67 Cache.HeaderP = (pkgCache::Header *)Map.Data();
95278287
DK
68 _error->PushToStack();
69 Map.RawAllocate(sizeof(pkgCache::Header));
70 bool const newError = _error->PendingError();
71 _error->MergeWithStack();
72 if (newError)
a133f79c
DK
73 return false;
74 if (Map.Size() <= 0)
75 return false;
c5f44afc 76
b2e465d6 77 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
c5f44afc 78
b2e465d6 79 // Starting header
578bfd0a 80 *Cache.HeaderP = pkgCache::Header();
e8a7b0b2
DK
81
82 // make room for the hashtables for packages and groups
32ab4bd0 83 if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
a133f79c 84 return false;
e8a7b0b2 85
4ad8619b 86 map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
7a223b93 87 if (unlikely(idxVerSysName == 0))
a133f79c 88 return false;
78a5476f 89 map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
7a223b93 90 if (unlikely(idxArchitecture == 0))
a133f79c
DK
91 return false;
92 map_stringitem_t idxArchitectures;
7a223b93
DK
93
94 std::vector<std::string> archs = APT::Configuration::getArchitectures();
95 if (archs.size() > 1)
96 {
97 std::vector<std::string>::const_iterator a = archs.begin();
98 std::string list = *a;
99 for (++a; a != archs.end(); ++a)
100 list.append(",").append(*a);
a133f79c 101 idxArchitectures = WriteStringInMap(list);
7a223b93 102 if (unlikely(idxArchitectures == 0))
a133f79c 103 return false;
7a223b93
DK
104 }
105 else
a133f79c
DK
106 idxArchitectures = idxArchitecture;
107
108 Cache.HeaderP = (pkgCache::Header *)Map.Data();
109 Cache.HeaderP->VerSysName = idxVerSysName;
110 Cache.HeaderP->Architecture = idxArchitecture;
111 Cache.HeaderP->SetArchitectures(idxArchitectures);
7a223b93 112
25c7a09d
JAK
113 // Calculate the hash for the empty map, so ReMap does not fail
114 Cache.HeaderP->CacheFileSize = Cache.CacheHash();
c5f44afc 115 Cache.ReMap();
578bfd0a 116 }
b2e465d6
AL
117 else
118 {
119 // Map directly from the existing file
120 Cache.ReMap();
121 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
122 if (Cache.VS != _system->VS)
a133f79c 123 return _error->Error(_("Cache has an incompatible versioning system"));
b2e465d6 124 }
e8a7b0b2 125
578bfd0a
AL
126 Cache.HeaderP->Dirty = true;
127 Map.Sync(0,sizeof(pkgCache::Header));
a133f79c 128 return true;
578bfd0a
AL
129}
130 /*}}}*/
131// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
132// ---------------------------------------------------------------------
133/* We sync the data then unset the dirty flag in two steps so as to
134 advoid a problem during a crash */
135pkgCacheGenerator::~pkgCacheGenerator()
136{
6789e01e 137 if (_error->PendingError() == true || Map.validData() == false)
578bfd0a
AL
138 return;
139 if (Map.Sync() == false)
140 return;
141
142 Cache.HeaderP->Dirty = false;
25c7a09d
JAK
143 Cache.HeaderP->CacheFileSize = Cache.CacheHash();
144
145 if (_config->FindB("Debug::pkgCacheGen", false))
146 std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
578bfd0a
AL
147 Map.Sync(0,sizeof(pkgCache::Header));
148}
149 /*}}}*/
19819ac5 150void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
9e7f8353
JAK
151 // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
152 // returns (something, true) the iterator was not yet seen and we can
153 // remap it.
154 std::unordered_set<void *> seen;
a9fe5928
DK
155 if (oldMap == newMap)
156 return;
157
64dda04b
MV
158 if (_config->FindB("Debug::pkgCacheGen", false))
159 std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
160
a9fe5928
DK
161 Cache.ReMap(false);
162
cf4ff3b7 163 CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
b07aeb1a 164 CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
a9fe5928 165
7635093c 166 for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
a9fe5928 167 i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
9e7f8353
JAK
168 if (std::get<1>(seen.insert(*i)) == true)
169 (*i)->ReMap(oldMap, newMap);
7635093c 170 for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
a9fe5928 171 i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
9e7f8353
JAK
172 if (std::get<1>(seen.insert(*i)) == true)
173 (*i)->ReMap(oldMap, newMap);
7635093c 174 for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
a9fe5928 175 i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
9e7f8353
JAK
176 if (std::get<1>(seen.insert(*i)) == true)
177 (*i)->ReMap(oldMap, newMap);
7635093c 178 for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
a9fe5928 179 i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
9e7f8353
JAK
180 if (std::get<1>(seen.insert(*i)) == true)
181 (*i)->ReMap(oldMap, newMap);
7635093c 182 for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
a9fe5928 183 i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
9e7f8353
JAK
184 if (std::get<1>(seen.insert(*i)) == true)
185 (*i)->ReMap(oldMap, newMap);
7635093c 186 for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
a9fe5928 187 i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
9e7f8353
JAK
188 if (std::get<1>(seen.insert(*i)) == true)
189 (*i)->ReMap(oldMap, newMap);
7635093c 190 for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
a9fe5928 191 i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
9e7f8353
JAK
192 if (std::get<1>(seen.insert(*i)) == true)
193 (*i)->ReMap(oldMap, newMap);
b07aeb1a
DK
194 for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
195 i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
9e7f8353
JAK
196 if (std::get<1>(seen.insert(*i)) == true)
197 (*i)->ReMap(oldMap, newMap);
7dd0c2eb 198 for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
9e7f8353
JAK
199 if (std::get<1>(seen.insert(ViewP)) == false)
200 continue;
7dd0c2eb
JAK
201 // Ignore views outside of the cache.
202 if (ViewP->data() < static_cast<const char*>(oldMap)
203 || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
204 continue;
205 const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
206 *ViewP = StringView(data , ViewP->size());
207 }
a9fe5928 208} /*}}}*/
7e58ab0c 209// CacheGenerator::WriteStringInMap /*{{{*/
4ad8619b 210map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
7e58ab0c 211 const unsigned long &Len) {
19819ac5 212 size_t oldSize = Map.Size();
a9fe5928 213 void const * const oldMap = Map.Data();
4ad8619b 214 map_stringitem_t const index = Map.WriteString(String, Len);
a9fe5928 215 if (index != 0)
19819ac5 216 ReMap(oldMap, Map.Data(), oldSize);
a9fe5928 217 return index;
7e58ab0c
DK
218}
219 /*}}}*/
220// CacheGenerator::WriteStringInMap /*{{{*/
4ad8619b 221map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
19819ac5 222 size_t oldSize = Map.Size();
a9fe5928 223 void const * const oldMap = Map.Data();
4ad8619b 224 map_stringitem_t const index = Map.WriteString(String);
a9fe5928 225 if (index != 0)
19819ac5 226 ReMap(oldMap, Map.Data(), oldSize);
a9fe5928 227 return index;
7e58ab0c
DK
228}
229 /*}}}*/
4ad8619b 230map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
19819ac5 231 size_t oldSize = Map.Size();
a9fe5928 232 void const * const oldMap = Map.Data();
4ad8619b 233 map_pointer_t const index = Map.Allocate(size);
a9fe5928 234 if (index != 0)
19819ac5 235 ReMap(oldMap, Map.Data(), oldSize);
a9fe5928 236 return index;
7e58ab0c
DK
237}
238 /*}}}*/
578bfd0a
AL
239// CacheGenerator::MergeList - Merge the package list /*{{{*/
240// ---------------------------------------------------------------------
241/* This provides the generation of the entries in the cache. Each loop
242 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
243bool pkgCacheGenerator::MergeList(ListParser &List,
244 pkgCache::VerIterator *OutVer)
578bfd0a
AL
245{
246 List.Owner = this;
0149949b 247
f9eec0e7 248 unsigned int Counter = 0;
0149949b 249 while (List.Step() == true)
578bfd0a 250 {
5bf15716 251 string const PackageName = List.Package();
65a1e968
AL
252 if (PackageName.empty() == true)
253 return false;
5bf15716 254
99a2ea5a
DK
255 Counter++;
256 if (Counter % 100 == 0 && Progress != 0)
257 Progress->Progress(List.Offset());
258
8efd5947
DK
259 APT::StringView Arch = List.Architecture();
260 Dynamic<APT::StringView> DynArch(Arch);
261 APT::StringView Version = List.Version();
262 Dynamic<APT::StringView> DynVersion(Version);
99a2ea5a
DK
263 if (Version.empty() == true && Arch.empty() == true)
264 {
8ec00880 265 // package descriptions
99a2ea5a
DK
266 if (MergeListGroup(List, PackageName) == false)
267 return false;
8ec00880 268 continue;
99a2ea5a
DK
269 }
270
8b32e920 271 // Get a pointer to the package structure
9ddf7030 272 pkgCache::PkgIterator Pkg;
a9fe5928 273 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
28166356 274 if (NewPackage(Pkg, PackageName, Arch) == false)
79e9003c
DK
275 // TRANSLATOR: The first placeholder is a package name,
276 // the other two should be copied verbatim as they include debug info
277 return _error->Error(_("Error occurred while processing %s (%s%d)"),
278 PackageName.c_str(), "NewPackage", 1);
8ce4327b 279
99a2ea5a 280
f55a958f
AL
281 if (Version.empty() == true)
282 {
99a2ea5a
DK
283 if (MergeListPackage(List, Pkg) == false)
284 return false;
285 }
286 else
287 {
288 if (MergeListVersion(List, Pkg, Version, OutVer) == false)
289 return false;
290 }
770c32ec 291
99a2ea5a 292 if (OutVer != 0)
99a2ea5a 293 return true;
99a2ea5a
DK
294 }
295
4ad8619b 296 if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
297 return _error->Error(_("Wow, you exceeded the number of package "
298 "names this APT is capable of."));
4ad8619b 299 if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
300 return _error->Error(_("Wow, you exceeded the number of versions "
301 "this APT is capable of."));
4ad8619b 302 if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
303 return _error->Error(_("Wow, you exceeded the number of descriptions "
304 "this APT is capable of."));
4ad8619b 305 if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
99a2ea5a
DK
306 return _error->Error(_("Wow, you exceeded the number of dependencies "
307 "this APT is capable of."));
308
99a2ea5a
DK
309 return true;
310}
311// CacheGenerator::MergeListGroup /*{{{*/
312bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
313{
314 pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
5f4db009
DK
315 // a group has no data on it's own, only packages have it but these
316 // stanzas like this come from Translation- files to add descriptions,
317 // but without a version we don't need a description for it…
99a2ea5a 318 if (Grp.end() == true)
5f4db009 319 return true;
99a2ea5a
DK
320 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
321
322 pkgCache::PkgIterator Pkg;
323 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
324 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
325 if (MergeListPackage(List, Pkg) == false)
326 return false;
327
328 return true;
329}
330 /*}}}*/
331// CacheGenerator::MergeListPackage /*{{{*/
332bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
333{
334 // we first process the package, then the descriptions
335 // (for deb this package processing is in fact a no-op)
336 pkgCache::VerIterator Ver(Cache);
337 Dynamic<pkgCache::VerIterator> DynVer(Ver);
338 if (List.UsePackage(Pkg, Ver) == false)
79e9003c
DK
339 return _error->Error(_("Error occurred while processing %s (%s%d)"),
340 Pkg.Name(), "UsePackage", 1);
99a2ea5a
DK
341
342 // Find the right version to write the description
343 MD5SumValue CurMd5 = List.Description_md5();
ffe3c68e 344 std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
99a2ea5a
DK
345 for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
346 {
a65e22c6 347 pkgCache::DescIterator VerDesc = Ver.DescriptionList();
22f07fc5
DK
348
349 // a version can only have one md5 describing it
a65e22c6 350 if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
22f07fc5 351 continue;
99a2ea5a 352
4ad8619b 353 map_stringitem_t md5idx = VerDesc->md5sum;
ffe3c68e
DK
354 for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
355 {
356 // don't add a new description if we have one for the given
357 // md5 && language
358 if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
359 continue;
a65e22c6 360
ffe3c68e
DK
361 AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
362 }
22f07fc5
DK
363
364 // we can stop here as all "same" versions will share the description
365 break;
99a2ea5a
DK
366 }
367
368 return true;
369}
370 /*}}}*/
371// CacheGenerator::MergeListVersion /*{{{*/
372bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
8efd5947 373 APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
99a2ea5a
DK
374{
375 pkgCache::VerIterator Ver = Pkg.VersionList();
376 Dynamic<pkgCache::VerIterator> DynVer(Ver);
4ad8619b 377 map_pointer_t *LastVer = &Pkg->VersionList;
99a2ea5a
DK
378 void const * oldMap = Map.Data();
379
68134bda 380 unsigned short const Hash = List.VersionHash();
99a2ea5a
DK
381 if (Ver.end() == false)
382 {
383 /* We know the list is sorted so we use that fact in the search.
384 Insertion of new versions is done with correct sorting */
2246928b 385 int Res = 1;
804de19f 386 for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
578bfd0a 387 {
8efd5947
DK
388 char const * const VerStr = Ver.VerStr();
389 Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
390 VerStr, VerStr + strlen(VerStr));
7e2b56a3
DK
391 // Version is higher as current version - insert here
392 if (Res > 0)
578bfd0a 393 break;
7e2b56a3 394 // Versionstrings are equal - is hash also equal?
cb9ac09b
DK
395 if (Res == 0)
396 {
397 if (List.SameVersion(Hash, Ver) == true)
398 break;
399 // sort (volatile) sources above not-sources like the status file
400 if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
401 {
402 auto VF = Ver.FileList();
403 for (; VF.end() == false; ++VF)
404 if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
405 break;
406 if (VF.end() == true)
407 break;
408 }
409 }
7e2b56a3
DK
410 // proceed with the next till we have either the right
411 // or we found another version (which will be lower)
578bfd0a 412 }
7e2b56a3
DK
413
414 /* We already have a version for this item, record that we saw it */
e426a5ff 415 if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
578bfd0a 416 {
f55a958f 417 if (List.UsePackage(Pkg,Ver) == false)
79e9003c
DK
418 return _error->Error(_("Error occurred while processing %s (%s%d)"),
419 Pkg.Name(), "UsePackage", 2);
f78439bf 420
578bfd0a 421 if (NewFileVer(Ver,List) == false)
79e9003c
DK
422 return _error->Error(_("Error occurred while processing %s (%s%d)"),
423 Pkg.Name(), "NewFileVer", 1);
99a2ea5a 424
ddc1d8d0
AL
425 // Read only a single record and return
426 if (OutVer != 0)
427 {
428 *OutVer = Ver;
429 return true;
430 }
99a2ea5a
DK
431
432 return true;
204fbdcc 433 }
99a2ea5a 434 }
204fbdcc 435
99a2ea5a 436 // Add a new version
4ad8619b 437 map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
95278287 438 if (unlikely(verindex == 0))
79e9003c
DK
439 return _error->Error(_("Error occurred while processing %s (%s%d)"),
440 Pkg.Name(), "NewVersion", 1);
a9fe5928 441
99a2ea5a 442 if (oldMap != Map.Data())
4ad8619b 443 LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
99a2ea5a 444 *LastVer = verindex;
a52f938b 445
79e9003c
DK
446 if (unlikely(List.NewVersion(Ver) == false))
447 return _error->Error(_("Error occurred while processing %s (%s%d)"),
448 Pkg.Name(), "NewVersion", 2);
a9fe5928 449
79e9003c
DK
450 if (unlikely(List.UsePackage(Pkg,Ver) == false))
451 return _error->Error(_("Error occurred while processing %s (%s%d)"),
452 Pkg.Name(), "UsePackage", 3);
453
454 if (unlikely(NewFileVer(Ver,List) == false))
455 return _error->Error(_("Error occurred while processing %s (%s%d)"),
456 Pkg.Name(), "NewFileVer", 2);
a52f938b 457
5a8e963b
DK
458 pkgCache::GrpIterator Grp = Pkg.Group();
459 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
460
461 /* If it is the first version of this package we need to add implicit
462 Multi-Arch dependencies to all other package versions in the group now -
463 otherwise we just add them for this new version */
464 if (Pkg.VersionList()->NextVer == 0)
465 {
466 pkgCache::PkgIterator P = Grp.PackageList();
467 Dynamic<pkgCache::PkgIterator> DynP(P);
468 for (; P.end() != true; P = Grp.NextPkg(P))
469 {
470 if (P->ID == Pkg->ID)
471 continue;
472 pkgCache::VerIterator V = P.VersionList();
473 Dynamic<pkgCache::VerIterator> DynV(V);
474 for (; V.end() != true; ++V)
475 if (unlikely(AddImplicitDepends(V, Pkg) == false))
476 return _error->Error(_("Error occurred while processing %s (%s%d)"),
477 Pkg.Name(), "AddImplicitDepends", 1);
478 }
479 }
480 if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
481 return _error->Error(_("Error occurred while processing %s (%s%d)"),
482 Pkg.Name(), "AddImplicitDepends", 2);
a52f938b 483
99a2ea5a
DK
484 // Read only a single record and return
485 if (OutVer != 0)
486 {
487 *OutVer = Ver;
488 return true;
578bfd0a 489 }
0149949b 490
ffe3c68e 491 /* Record the Description(s) based on their master md5sum */
5f4db009 492 MD5SumValue CurMd5 = List.Description_md5();
5f4db009
DK
493
494 /* Before we add a new description we first search in the group for
495 a version with a description of the same MD5 - if so we reuse this
496 description group instead of creating our own for this version */
5f4db009
DK
497 for (pkgCache::PkgIterator P = Grp.PackageList();
498 P.end() == false; P = Grp.NextPkg(P))
499 {
500 for (pkgCache::VerIterator V = P.VersionList();
501 V.end() == false; ++V)
502 {
ffe3c68e 503 if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
5f4db009
DK
504 continue;
505 Ver->DescriptionList = V->DescriptionList;
5f4db009
DK
506 }
507 }
508
ffe3c68e 509 // We haven't found reusable descriptions, so add the first description(s)
4ad8619b 510 map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
ffe3c68e
DK
511 std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
512 for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
513 if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
514 return false;
515 return true;
516}
517 /*}}}*/
4ad8619b 518bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
ffe3c68e
DK
519{
520 pkgCache::DescIterator Desc;
99a2ea5a 521 Dynamic<pkgCache::DescIterator> DynDesc(Desc);
99a2ea5a 522
4ad8619b 523 map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
95278287 524 if (unlikely(descindex == 0))
5954d4d2 525 return _error->Error(_("Error occurred while processing %s (%s%d)"),
ffe3c68e
DK
526 Ver.ParentPkg().Name(), "NewDescription", 1);
527
528 md5idx = Desc->md5sum;
529 Desc->ParentPkg = Ver.ParentPkg().Index();
a65e22c6 530
ffe3c68e
DK
531 // we add at the end, so that the start is constant as we need
532 // that to be able to efficiently share these lists
533 pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
534 for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
4ad8619b 535 map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
ffe3c68e 536 *LastNextDesc = descindex;
99a2ea5a 537
a65e22c6 538 if (NewFileDesc(Desc,List) == false)
79e9003c 539 return _error->Error(_("Error occurred while processing %s (%s%d)"),
ffe3c68e 540 Ver.ParentPkg().Name(), "NewFileDesc", 1);
45415543 541
578bfd0a
AL
542 return true;
543}
544 /*}}}*/
99a2ea5a 545 /*}}}*/
5bf15716
DK
546// CacheGenerator::NewGroup - Add a new group /*{{{*/
547// ---------------------------------------------------------------------
548/* This creates a new group structure and adds it to the hash table */
eff0c22e 549bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
33dd02e3 550{
7dd0c2eb 551 Dynamic<StringView> DName(Name);
33dd02e3
DK
552 Grp = Cache.FindGrp(Name);
553 if (Grp.end() == false)
554 return true;
5bf15716 555
33dd02e3 556 // Get a structure
4ad8619b 557 map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
33dd02e3
DK
558 if (unlikely(Group == 0))
559 return false;
5bf15716 560
33dd02e3 561 Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
dd592790 562 map_stringitem_t const idxName = StoreString(PKGNAME, Name);
a9fe5928 563 if (unlikely(idxName == 0))
33dd02e3 564 return false;
a9fe5928 565 Grp->Name = idxName;
5bf15716 566
33dd02e3
DK
567 // Insert it into the hash table
568 unsigned long const Hash = Cache.Hash(Name);
32ab4bd0 569 map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
60d523e2
JAK
570
571 while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
aa0fe657
DK
572 insertAt = &(Cache.GrpP + *insertAt)->Next;
573 Grp->Next = *insertAt;
574 *insertAt = Group;
5bf15716 575
52c41485 576 Grp->ID = Cache.HeaderP->GroupCount++;
33dd02e3 577 return true;
5bf15716
DK
578}
579 /*}}}*/
578bfd0a
AL
580// CacheGenerator::NewPackage - Add a new package /*{{{*/
581// ---------------------------------------------------------------------
582/* This creates a new package structure and adds it to the hash table */
eff0c22e
JAK
583bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
584 StringView Arch) {
5bf15716 585 pkgCache::GrpIterator Grp;
7dd0c2eb
JAK
586 Dynamic<StringView> DName(Name);
587 Dynamic<StringView> DArch(Arch);
a9fe5928 588 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
5bf15716
DK
589 if (unlikely(NewGroup(Grp, Name) == false))
590 return false;
591
592 Pkg = Grp.FindPkg(Arch);
593 if (Pkg.end() == false)
594 return true;
a52f938b 595
578bfd0a 596 // Get a structure
4ad8619b 597 map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
5bf15716 598 if (unlikely(Package == 0))
578bfd0a 599 return false;
f55a958f 600 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
5bf15716 601
ecc138f8
DK
602 // Set the name, arch and the ID
603 APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
604 Pkg->Group = Grp.Index();
605 // all is mapped to the native architecture
dd592790 606 map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
ecc138f8
DK
607 if (unlikely(idxArch == 0))
608 return false;
609 Pkg->Arch = idxArch;
610 Pkg->ID = Cache.HeaderP->PackageCount++;
611
c408e01e
DK
612 // Insert the package into our package list
613 if (Grp->FirstPackage == 0) // the group is new
614 {
aa0fe657 615 Grp->FirstPackage = Package;
c408e01e 616 // Insert it into the hash table
4ad8619b 617 map_id_t const Hash = Cache.Hash(Name);
32ab4bd0 618 map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
eff0c22e 619 while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
32ab4bd0
DK
620 insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
621 Pkg->NextPackage = *insertAt;
aa0fe657 622 *insertAt = Package;
c408e01e
DK
623 }
624 else // Group the Packages together
625 {
62428dbc 626 // if sibling is provided by another package, this one is too
ecc138f8
DK
627 {
628 pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
a6deb7a0
JAK
629 if (M.end() == false) {
630 pkgCache::PrvIterator Prv;
631 Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
632 for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
ecc138f8
DK
633 {
634 if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
635 continue;
636 pkgCache::VerIterator Ver = Prv.OwnerVer();
a6deb7a0 637 Dynamic<pkgCache::VerIterator> DynVer(Ver);
ecc138f8
DK
638 if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
639 ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
640 (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
62428dbc
DK
641 {
642 if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
643 continue;
ecc138f8
DK
644 if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
645 return false;
62428dbc 646 }
ecc138f8 647 }
a6deb7a0 648 }
62428dbc
DK
649 }
650 // let M-A:foreign package siblings provide this package
651 {
a6deb7a0
JAK
652 pkgCache::PkgIterator P;
653 pkgCache::VerIterator Ver;
654 Dynamic<pkgCache::PkgIterator> DynP(P);
655 Dynamic<pkgCache::VerIterator> DynVer(Ver);
ecc138f8 656
a6deb7a0 657 for (P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
62428dbc
DK
658 {
659 if (APT::Configuration::checkArchitecture(P.Arch()) == false)
660 continue;
a6deb7a0 661 for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
ecc138f8
DK
662 if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
663 if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
664 return false;
62428dbc 665 }
ecc138f8 666 }
bb0f6a34
DK
667 // and negative dependencies, don't forget negative dependencies
668 {
669 pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
05c001c8
JAK
670 if (M.end() == false) {
671 pkgCache::DepIterator Dep;
672 Dynamic<pkgCache::DepIterator> DynDep(Dep);
673 for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
bb0f6a34
DK
674 {
675 if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
676 continue;
677 if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
678 Dep->Type != pkgCache::Dep::Replaces)
679 continue;
680 pkgCache::VerIterator Ver = Dep.ParentVer();
05c001c8 681 Dynamic<pkgCache::VerIterator> DynVer(Ver);
bb0f6a34
DK
682 map_pointer_t * unused = NULL;
683 if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
684 return false;
685 }
05c001c8 686 }
bb0f6a34 687 }
ecc138f8 688
c408e01e
DK
689 // this package is the new last package
690 pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
32ab4bd0
DK
691 Pkg->NextPackage = LastPkg->NextPackage;
692 LastPkg->NextPackage = Package;
c408e01e
DK
693 }
694 Grp->LastPackage = Package;
3addaba1
DK
695
696 // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
697 if (Arch == "any")
698 {
074564d4 699 size_t const found = Name.rfind(':');
8ee09b58 700 StringView ArchA = Name.substr(found + 1);
530302ef 701 if (ArchA != "any")
3addaba1 702 {
8ee09b58
JAK
703 // ArchA is used inside the loop which might remap (NameA is not used)
704 Dynamic<StringView> DynArchA(ArchA);
530302ef
DK
705 StringView NameA = Name.substr(0, found);
706 pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
3addaba1 707 Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
530302ef 708 if (PkgA.end())
3addaba1 709 {
530302ef
DK
710 Dynamic<StringView> DynNameA(NameA);
711 if (NewPackage(PkgA, NameA, ArchA) == false)
3addaba1
DK
712 return false;
713 }
530302ef
DK
714 if (unlikely(PkgA.end()))
715 return _error->Fatal("NewPackage was successful for %s:%s,"
716 "but the package doesn't exist anyhow!",
717 NameA.to_string().c_str(), ArchA.to_string().c_str());
718 else
3addaba1 719 {
530302ef
DK
720 pkgCache::PrvIterator Prv = PkgA.ProvidesList();
721 for (; Prv.end() == false; ++Prv)
722 {
723 if (Prv.IsMultiArchImplicit())
724 continue;
725 pkgCache::VerIterator V = Prv.OwnerVer();
726 if (ArchA != V.ParentPkg().Arch())
727 continue;
728 if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
729 return false;
730 }
731 pkgCache::VerIterator V = PkgA.VersionList();
732 Dynamic<pkgCache::VerIterator> DynV(V);
733 for (; V.end() == false; ++V)
734 {
735 if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
736 return false;
737 }
3addaba1
DK
738 }
739 }
740 }
578bfd0a
AL
741 return true;
742}
5a8e963b
DK
743 /*}}}*/
744// CacheGenerator::AddImplicitDepends /*{{{*/
745bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
746 pkgCache::PkgIterator &P,
747 pkgCache::VerIterator &V)
748{
8efd5947
DK
749 APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
750 Dynamic<APT::StringView> DynArch(Arch);
4ad8619b 751 map_pointer_t *OldDepLast = NULL;
5a8e963b
DK
752 /* MultiArch handling introduces a lot of implicit Dependencies:
753 - MultiArch: same → Co-Installable if they have the same version
754 - All others conflict with all other group members */
755 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
756 pkgCache::PkgIterator D = G.PackageList();
757 Dynamic<pkgCache::PkgIterator> DynD(D);
4ad8619b 758 map_stringitem_t const VerStrIdx = V->VerStr;
5a8e963b
DK
759 for (; D.end() != true; D = G.NextPkg(D))
760 {
761 if (Arch == D.Arch() || D->VersionList == 0)
762 continue;
763 /* We allow only one installed arch at the time
764 per group, therefore each group member conflicts
765 with all other group members */
766 if (coInstall == true)
767 {
768 // Replaces: ${self}:other ( << ${binary:Version})
9c44383f 769 NewDepends(D, V, VerStrIdx,
8c7af4d4 770 pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
5a8e963b
DK
771 OldDepLast);
772 // Breaks: ${self}:other (!= ${binary:Version})
9c44383f 773 NewDepends(D, V, VerStrIdx,
8c7af4d4 774 pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
5a8e963b
DK
775 OldDepLast);
776 } else {
777 // Conflicts: ${self}:other
9c44383f 778 NewDepends(D, V, 0,
8c7af4d4 779 pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
5a8e963b
DK
780 OldDepLast);
781 }
782 }
783 return true;
784}
785bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
786 pkgCache::PkgIterator &D)
787{
788 /* MultiArch handling introduces a lot of implicit Dependencies:
789 - MultiArch: same → Co-Installable if they have the same version
790 - All others conflict with all other group members */
4ad8619b 791 map_pointer_t *OldDepLast = NULL;
5a8e963b
DK
792 bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
793 if (coInstall == true)
794 {
4ad8619b 795 map_stringitem_t const VerStrIdx = V->VerStr;
5a8e963b 796 // Replaces: ${self}:other ( << ${binary:Version})
9c44383f 797 NewDepends(D, V, VerStrIdx,
8c7af4d4 798 pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
5a8e963b
DK
799 OldDepLast);
800 // Breaks: ${self}:other (!= ${binary:Version})
9c44383f 801 NewDepends(D, V, VerStrIdx,
8c7af4d4 802 pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
5a8e963b
DK
803 OldDepLast);
804 } else {
805 // Conflicts: ${self}:other
9c44383f 806 NewDepends(D, V, 0,
8c7af4d4 807 pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
5a8e963b
DK
808 OldDepLast);
809 }
810 return true;
811}
812
578bfd0a
AL
813 /*}}}*/
814// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
815// ---------------------------------------------------------------------
816/* */
f55a958f 817bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
818 ListParser &List)
819{
ddc1d8d0
AL
820 if (CurrentFile == 0)
821 return true;
822
dcb79bae 823 // Get a structure
4ad8619b 824 map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
dcb79bae 825 if (VerFile == 0)
ac2c559b 826 return false;
dcb79bae
AL
827
828 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
829 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
830
831 // Link it to the end of the list
4ad8619b 832 map_pointer_t *Last = &Ver->FileList;
f7f0d6c7 833 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
03e39e59
AL
834 Last = &V->NextFile;
835 VF->NextFile = *Last;
836 *Last = VF.Index();
837
dcb79bae
AL
838 VF->Offset = List.Offset();
839 VF->Size = List.Size();
ad00ae81
AL
840 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
841 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
842 Cache.HeaderP->VerFileCount++;
843
f55a958f 844 return true;
578bfd0a
AL
845}
846 /*}}}*/
847// CacheGenerator::NewVersion - Create a new Version /*{{{*/
848// ---------------------------------------------------------------------
f55a958f 849/* This puts a version structure in the linked list */
4ad8619b 850map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
8efd5947 851 APT::StringView const &VerStr,
4ad8619b
DK
852 map_pointer_t const ParentPkg,
853 unsigned short const Hash,
854 map_pointer_t const Next)
578bfd0a 855{
f55a958f 856 // Get a structure
4ad8619b 857 map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
f55a958f 858 if (Version == 0)
0149949b 859 return 0;
f55a958f
AL
860
861 // Fill it in
862 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
dfe45e1f 863 //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
f55a958f 864 Ver->NextVer = Next;
885594fc
DK
865 Ver->ParentPkg = ParentPkg;
866 Ver->Hash = Hash;
f55a958f 867 Ver->ID = Cache.HeaderP->VersionCount++;
885594fc
DK
868
869 // try to find the version string in the group for reuse
870 pkgCache::PkgIterator Pkg = Ver.ParentPkg();
871 pkgCache::GrpIterator Grp = Pkg.Group();
872 if (Pkg.end() == false && Grp.end() == false)
873 {
874 for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
875 {
876 if (Pkg == P)
877 continue;
878 for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
879 {
8efd5947
DK
880 int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
881 if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
885594fc
DK
882 {
883 Ver->VerStr = V->VerStr;
884 return Version;
885 }
886 else if (cmp < 0)
887 break;
888 }
889 }
890 }
891 // haven't found the version string, so create
30b45652 892 map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
a9fe5928 893 if (unlikely(idxVerStr == 0))
0149949b 894 return 0;
a9fe5928 895 Ver->VerStr = idxVerStr;
0149949b 896 return Version;
578bfd0a
AL
897}
898 /*}}}*/
a52f938b
OS
899// CacheGenerator::NewFileDesc - Create a new File<->Desc association /*{{{*/
900// ---------------------------------------------------------------------
901/* */
902bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
903 ListParser &List)
904{
905 if (CurrentFile == 0)
906 return true;
907
908 // Get a structure
4ad8619b 909 map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
a52f938b 910 if (DescFile == 0)
c5f44afc 911 return false;
770c32ec 912
a52f938b
OS
913 pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
914 DF->File = CurrentFile - Cache.PkgFileP;
770c32ec 915
a52f938b 916 // Link it to the end of the list
4ad8619b 917 map_pointer_t *Last = &Desc->FileList;
f7f0d6c7 918 for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
a52f938b 919 Last = &D->NextFile;
770c32ec 920
a52f938b
OS
921 DF->NextFile = *Last;
922 *Last = DF.Index();
923
924 DF->Offset = List.Offset();
925 DF->Size = List.Size();
926 if (Cache.HeaderP->MaxDescFileSize < DF->Size)
927 Cache.HeaderP->MaxDescFileSize = DF->Size;
928 Cache.HeaderP->DescFileCount++;
929
930 return true;
931}
932 /*}}}*/
933// CacheGenerator::NewDescription - Create a new Description /*{{{*/
934// ---------------------------------------------------------------------
935/* This puts a description structure in the linked list */
4ad8619b 936map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
a65e22c6
DK
937 const string &Lang,
938 const MD5SumValue &md5sum,
4ad8619b 939 map_stringitem_t const idxmd5str)
a52f938b
OS
940{
941 // Get a structure
4ad8619b 942 map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
a52f938b
OS
943 if (Description == 0)
944 return 0;
945
946 // Fill it in
947 Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
a52f938b 948 Desc->ID = Cache.HeaderP->DescriptionCount++;
78a5476f 949 map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
a65e22c6 950 if (unlikely(idxlanguage_code == 0))
c5f44afc 951 return 0;
a9fe5928 952 Desc->language_code = idxlanguage_code;
a65e22c6
DK
953
954 if (idxmd5str != 0)
955 Desc->md5sum = idxmd5str;
956 else
957 {
4ad8619b 958 map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
a65e22c6
DK
959 if (unlikely(idxmd5sum == 0))
960 return 0;
961 Desc->md5sum = idxmd5sum;
962 }
a52f938b
OS
963
964 return Description;
965}
966 /*}}}*/
25396fb0 967// CacheGenerator::NewDepends - Create a dependency element /*{{{*/
dcb79bae
AL
968// ---------------------------------------------------------------------
969/* This creates a dependency element in the tree. It is linked to the
970 version and to the package that it is pointing to. */
9c44383f
DK
971bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
972 pkgCache::VerIterator &Ver,
4ad8619b 973 map_pointer_t const Version,
8c7af4d4
DK
974 uint8_t const Op,
975 uint8_t const Type,
4ad8619b 976 map_pointer_t* &OldDepLast)
dcb79bae 977{
a9fe5928 978 void const * const oldMap = Map.Data();
dcb79bae 979 // Get a structure
4ad8619b 980 map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
5bf15716 981 if (unlikely(Dependency == 0))
dcb79bae 982 return false;
9c44383f 983
71c9e95b
DK
984 bool isDuplicate = false;
985 map_pointer_t DependencyData = 0;
b291aa59
DK
986 map_pointer_t PreviousData = 0;
987 if (Pkg->RevDepends != 0)
71c9e95b 988 {
b291aa59
DK
989 pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
990 DependencyData = L->DependencyData;
991 do {
992 pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
993 if (Version > D->Version)
994 break;
995 if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
996 {
997 isDuplicate = true;
998 break;
999 }
1000 PreviousData = DependencyData;
1001 DependencyData = D->NextData;
1002 } while (DependencyData != 0);
71c9e95b
DK
1003 }
1004
1005 if (isDuplicate == false)
1006 {
71c9e95b
DK
1007 DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
1008 if (unlikely(DependencyData == 0))
1009 return false;
71c9e95b 1010 }
5bf15716 1011
71c9e95b
DK
1012 pkgCache::Dependency * Link = Cache.DepP + Dependency;
1013 Link->ParentVer = Ver.Index();
1014 Link->DependencyData = DependencyData;
1015 Link->ID = Cache.HeaderP->DependsCount++;
1016
1017 pkgCache::DepIterator Dep(Cache, Link);
71c9e95b
DK
1018 if (isDuplicate == false)
1019 {
1020 Dep->Type = Type;
1021 Dep->CompareOp = Op;
1022 Dep->Version = Version;
1023 Dep->Package = Pkg.Index();
1024 ++Cache.HeaderP->DependsDataCount;
b291aa59
DK
1025 if (PreviousData != 0)
1026 {
1027 pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1028 Dep->NextData = D->NextData;
1029 D->NextData = DependencyData;
1030 }
1031 else if (Pkg->RevDepends != 0)
1032 {
1033 pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1034 Dep->NextData = D->DependencyData;
1035 }
1036 }
1037
1038 if (isDuplicate == true || PreviousData != 0)
1039 {
1040 pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1041 Link->NextRevDepends = L->NextRevDepends;
1042 L->NextRevDepends = Dependency;
71c9e95b
DK
1043 }
1044 else
1045 {
b291aa59
DK
1046 Link->NextRevDepends = Pkg->RevDepends;
1047 Pkg->RevDepends = Dependency;
71c9e95b 1048 }
25396fb0 1049
b291aa59 1050
25396fb0
DK
1051 // Do we know where to link the Dependency to?
1052 if (OldDepLast == NULL)
c1a22377 1053 {
f9eec0e7 1054 OldDepLast = &Ver->DependsList;
f7f0d6c7 1055 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
f9eec0e7 1056 OldDepLast = &D->NextDepends;
a9fe5928 1057 } else if (oldMap != Map.Data())
4ad8619b 1058 OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
45415543 1059
f9eec0e7 1060 Dep->NextDepends = *OldDepLast;
71c9e95b 1061 *OldDepLast = Dependency;
f9eec0e7 1062 OldDepLast = &Dep->NextDepends;
dcb79bae
AL
1063 return true;
1064}
1065 /*}}}*/
25396fb0
DK
1066// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1067// ---------------------------------------------------------------------
1068/* This creates a Group and the Package to link this dependency to if
1069 needed and handles also the caching of the old endpoint */
c9443c01 1070bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
eff0c22e
JAK
1071 StringView PackageName,
1072 StringView Arch,
1073 StringView Version,
8c7af4d4
DK
1074 uint8_t const Op,
1075 uint8_t const Type)
25396fb0
DK
1076{
1077 pkgCache::GrpIterator Grp;
a9fe5928 1078 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
7dd0c2eb
JAK
1079 Dynamic<StringView> DynPackageName(PackageName);
1080 Dynamic<StringView> DynArch(Arch);
1081 Dynamic<StringView> DynVersion(Version);
25396fb0
DK
1082 if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1083 return false;
1084
bb0f6a34
DK
1085 map_stringitem_t idxVersion = 0;
1086 if (Version.empty() == false)
1087 {
1088 int const CmpOp = Op & 0x0F;
1089 // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
57590d3c 1090 if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
bb0f6a34
DK
1091 idxVersion = Ver->VerStr;
1092
1093 if (idxVersion == 0)
1094 {
dd592790 1095 idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
bb0f6a34
DK
1096 if (unlikely(idxVersion == 0))
1097 return false;
1098 }
25396fb0
DK
1099 }
1100
bb0f6a34
DK
1101 bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1102 Type == pkgCache::Dep::Conflicts ||
1103 Type == pkgCache::Dep::Replaces);
1104
1105 pkgCache::PkgIterator Pkg;
1106 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1107 if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1108 {
1109 // Locate the target package
1110 Pkg = Grp.FindPkg(Arch);
1111 if (Pkg.end() == true) {
1112 if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1113 return false;
1114 }
1115
1116 /* Caching the old end point speeds up generation substantially */
1117 if (OldDepVer != Ver) {
1118 OldDepLast = NULL;
1119 OldDepVer = Ver;
1120 }
1121
1122 return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1123 }
1124 else
1125 {
1126 /* Caching the old end point speeds up generation substantially */
1127 if (OldDepVer != Ver) {
1128 OldDepLast = NULL;
1129 OldDepVer = Ver;
1130 }
1131
1132 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1133 {
1134 if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1135 return false;
1136 }
1137 }
1138 return true;
25396fb0
DK
1139}
1140 /*}}}*/
dcb79bae 1141// ListParser::NewProvides - Create a Provides element /*{{{*/
c9443c01 1142bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
eff0c22e
JAK
1143 StringView PkgName,
1144 StringView PkgArch,
1145 StringView Version,
8c7af4d4 1146 uint8_t const Flags)
dcb79bae 1147{
ecc138f8 1148 pkgCache const &Cache = Owner->Cache;
7dd0c2eb
JAK
1149 Dynamic<StringView> DynPkgName(PkgName);
1150 Dynamic<StringView> DynArch(PkgArch);
1151 Dynamic<StringView> DynVersion(Version);
8efa2a3b
AL
1152
1153 // We do not add self referencing provides
566046f4 1154 if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
b755de25
DK
1155 (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1156 (Version.empty() || Version == Ver.VerStr()))
8efa2a3b 1157 return true;
ecc138f8
DK
1158
1159 // Locate the target package
1160 pkgCache::PkgIterator Pkg;
1161 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1162 if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1163 return false;
1164
1165 map_stringitem_t idxProvideVersion = 0;
1166 if (Version.empty() == false) {
dd592790 1167 idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
ecc138f8
DK
1168 if (unlikely(idxProvideVersion == 0))
1169 return false;
1170 }
1171 return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1172}
1173bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1174 pkgCache::PkgIterator &Pkg,
1175 map_pointer_t const ProvideVersion,
1176 uint8_t const Flags)
1177{
dcb79bae 1178 // Get a structure
ecc138f8 1179 map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
5bf15716 1180 if (unlikely(Provides == 0))
dcb79bae 1181 return false;
ecc138f8
DK
1182 ++Cache.HeaderP->ProvidesCount;
1183
dcb79bae
AL
1184 // Fill it in
1185 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1186 Prv->Version = Ver.Index();
ecc138f8 1187 Prv->ProvideVersion = ProvideVersion;
8c7af4d4 1188 Prv->Flags = Flags;
dcb79bae
AL
1189 Prv->NextPkgProv = Ver->ProvidesList;
1190 Ver->ProvidesList = Prv.Index();
ecc138f8 1191
dcb79bae
AL
1192 // Link it to the package
1193 Prv->ParentPkg = Pkg.Index();
1194 Prv->NextProvides = Pkg->ProvidesList;
1195 Pkg->ProvidesList = Prv.Index();
ecc138f8
DK
1196 return true;
1197}
1198 /*}}}*/
1199// ListParser::NewProvidesAllArch - add provides for all architectures /*{{{*/
eff0c22e
JAK
1200bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1201 StringView Version, uint8_t const Flags) {
ecc138f8 1202 pkgCache &Cache = Owner->Cache;
a6deb7a0
JAK
1203 pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1204 Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
7dd0c2eb
JAK
1205 Dynamic<StringView> DynPackage(Package);
1206 Dynamic<StringView> DynVersion(Version);
a6deb7a0 1207
ecc138f8
DK
1208 if (Grp.end() == true)
1209 return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1210 else
1211 {
1212 map_stringitem_t idxProvideVersion = 0;
1213 if (Version.empty() == false) {
dd592790 1214 idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
ecc138f8
DK
1215 if (unlikely(idxProvideVersion == 0))
1216 return false;
1217 }
1218
1219 bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1220 bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
a6deb7a0
JAK
1221 pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1222 Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1223 pkgCache::PkgIterator Pkg;
1224 Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1225 for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
ecc138f8
DK
1226 {
1227 if (isImplicit && OwnerPkg == Pkg)
1228 continue;
1229 if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1230 continue;
1231 if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1232 return false;
1233 }
1234 }
dcb79bae
AL
1235 return true;
1236}
1237 /*}}}*/
c9443c01 1238bool pkgCacheListParser::SameVersion(unsigned short const Hash, /*{{{*/
68134bda
DK
1239 pkgCache::VerIterator const &Ver)
1240{
1241 return Hash == Ver->Hash;
1242}
1243 /*}}}*/
b07aeb1a
DK
1244// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to /*{{{*/
1245bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1246 unsigned long Flags)
1247{
1248 if (File.empty() && Site.empty())
1249 {
1250 CurrentRlsFile = NULL;
1251 return true;
1252 }
1253
1254 // Get some space for the structure
1255 map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1256 if (unlikely(idxFile == 0))
1257 return false;
1258 CurrentRlsFile = Cache.RlsFileP + idxFile;
1259
1260 // Fill it in
1261 map_stringitem_t const idxFileName = WriteStringInMap(File);
1262 map_stringitem_t const idxSite = StoreString(MIXED, Site);
1263 if (unlikely(idxFileName == 0 || idxSite == 0))
1264 return false;
1265 CurrentRlsFile->FileName = idxFileName;
1266 CurrentRlsFile->Site = idxSite;
1267 CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1268 CurrentRlsFile->Flags = Flags;
1269 CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1270 RlsFileName = File;
1271 Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1272 Cache.HeaderP->ReleaseFileCount++;
1273
1274 return true;
1275}
1276 /*}}}*/
578bfd0a
AL
1277// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
1278// ---------------------------------------------------------------------
1279/* This is used to select which file is to be associated with all newly
b2e465d6 1280 added versions. The caller is responsible for setting the IMS fields. */
b07aeb1a
DK
1281bool pkgCacheGenerator::SelectFile(std::string const &File,
1282 pkgIndexFile const &Index,
e185d8b3 1283 std::string const &Architecture,
b07aeb1a
DK
1284 std::string const &Component,
1285 unsigned long const Flags)
578bfd0a 1286{
578bfd0a 1287 // Get some space for the structure
4ad8619b 1288 map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
a9fe5928 1289 if (unlikely(idxFile == 0))
578bfd0a 1290 return false;
a9fe5928
DK
1291 CurrentFile = Cache.PkgFileP + idxFile;
1292
578bfd0a 1293 // Fill it in
4ad8619b 1294 map_stringitem_t const idxFileName = WriteStringInMap(File);
b07aeb1a 1295 if (unlikely(idxFileName == 0))
a9fe5928
DK
1296 return false;
1297 CurrentFile->FileName = idxFileName;
578bfd0a 1298 CurrentFile->NextFile = Cache.HeaderP->FileList;
e1b74f61 1299 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
78a5476f 1300 map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
a9fe5928
DK
1301 if (unlikely(idxIndexType == 0))
1302 return false;
1303 CurrentFile->IndexType = idxIndexType;
e185d8b3
DK
1304 if (Architecture.empty())
1305 CurrentFile->Architecture = 0;
1306 else
1307 {
1308 map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1309 if (unlikely(arch == 0))
1310 return false;
1311 CurrentFile->Architecture = arch;
1312 }
b07aeb1a
DK
1313 map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1314 if (unlikely(component == 0))
1315 return false;
1316 CurrentFile->Component = component;
1317 CurrentFile->Flags = Flags;
1318 if (CurrentRlsFile != NULL)
1319 CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1320 else
1321 CurrentFile->Release = 0;
578bfd0a 1322 PkgFileName = File;
ad00ae81 1323 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 1324 Cache.HeaderP->PackageFileCount++;
b2e465d6 1325
ddc1d8d0 1326 if (Progress != 0)
b2e465d6 1327 Progress->SubProgress(Index.Size());
8efa2a3b 1328 return true;
578bfd0a
AL
1329}
1330 /*}}}*/
f55a958f
AL
1331// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
1332// ---------------------------------------------------------------------
1333/* This is used to create handles to strings. Given the same text it
1334 always returns the same number */
78a5476f 1335map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
f55a958f
AL
1336 unsigned int Size)
1337{
869700d8 1338 auto strings = &strMixed;
78a5476f
DK
1339 switch(type) {
1340 case MIXED: strings = &strMixed; break;
1341 case PKGNAME: strings = &strPkgNames; break;
30b45652 1342 case VERSIONNUMBER: strings = &strVersions; break;
78a5476f 1343 case SECTION: strings = &strSections; break;
869700d8 1344 default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
f55a958f 1345 }
a9fe5928 1346
869700d8 1347 auto const item = strings->find({S, Size, nullptr, 0});
78a5476f 1348 if (item != strings->end())
869700d8 1349 return item->item;
a9fe5928 1350
78a5476f 1351 map_stringitem_t const idxString = WriteStringInMap(S,Size);
869700d8 1352 strings->insert({nullptr, Size, this, idxString});
78a5476f 1353 return idxString;
f55a958f
AL
1354}
1355 /*}}}*/
b2e465d6 1356// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 1357// ---------------------------------------------------------------------
b2e465d6
AL
1358/* This just verifies that each file in the list of index files exists,
1359 has matching attributes with the cache and the cache does not have
1360 any extra files. */
95278287
DK
1361class APT_HIDDEN ScopedErrorRevert {
1362public:
1363 ScopedErrorRevert() { _error->PushToStack(); }
1364 ~ScopedErrorRevert() { _error->RevertToStack(); }
1365};
5465192b 1366static bool CheckValidity(const string &CacheFile,
2ec858bc 1367 pkgSourceList &List,
5465192b
DK
1368 FileIterator const Start,
1369 FileIterator const End,
f1616039
JAK
1370 MMap **OutMap = 0,
1371 pkgCache **OutCache = 0)
b35d2f5f 1372{
95278287 1373 ScopedErrorRevert ser;
c8e572e3 1374 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
b2e465d6
AL
1375 // No file, certainly invalid
1376 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
c8e572e3
MV
1377 {
1378 if (Debug == true)
5465192b 1379 std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
b35d2f5f 1380 return false;
c8e572e3
MV
1381 }
1382
96d14a91 1383 if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
2ec858bc
MV
1384 {
1385 if (Debug == true)
1386 std::clog << "sources.list is newer than the cache" << std::endl;
1387 return false;
1388 }
1389
b2e465d6 1390 // Map it
b35d2f5f 1391 FileFd CacheF(CacheFile,FileFd::ReadOnly);
6c413b18 1392 std::unique_ptr<MMap> Map(new MMap(CacheF,0));
6789e01e
DK
1393 if (unlikely(Map->validData()) == false)
1394 return false;
f1616039
JAK
1395 std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1396 pkgCache &Cache = *CacheP.get();
95278287 1397 if (_error->PendingError() || Map->Size() == 0)
b35d2f5f 1398 {
c8e572e3 1399 if (Debug == true)
5465192b 1400 std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
b35d2f5f
AL
1401 return false;
1402 }
b07aeb1a 1403
98cc7fd2
JAK
1404 std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1405 memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
b07aeb1a
DK
1406 std::vector<pkgIndexFile *> Files;
1407 for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1408 {
1409 if (Debug == true)
1410 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
3fd89e62 1411 pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
b07aeb1a
DK
1412 if (RlsFile.end() == true)
1413 {
1414 if (Debug == true)
1415 std::clog << "FindInCache returned end-Pointer" << std::endl;
1416 return false;
1417 }
1418
1419 RlsVisited[RlsFile->ID] = true;
1420 if (Debug == true)
1421 std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1422
5465192b
DK
1423 std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1424 std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1425 [](pkgIndexFile const * const I) { return I->HasPackages(); });
b07aeb1a
DK
1426 }
1427 for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1428 if (RlsVisited[I] == false)
1429 {
1430 if (Debug == true)
1431 std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1432 return false;
1433 }
1434
5465192b 1435 std::copy(Start, End, std::back_inserter(Files));
b07aeb1a 1436
b2e465d6
AL
1437 /* Now we check every index file, see if it is in the cache,
1438 verify the IMS data and check that it is on the disk too.. */
98cc7fd2
JAK
1439 std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1440 memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
b07aeb1a 1441 for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
c8e572e3
MV
1442 {
1443 if (Debug == true)
b07aeb1a
DK
1444 std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1445 if ((*PkgFile)->Exists() == false)
b35d2f5f 1446 {
c8e572e3
MV
1447 if (Debug == true)
1448 std::clog << "file doesn't exist" << std::endl;
b2e465d6 1449 continue;
b35d2f5f 1450 }
b2e465d6
AL
1451
1452 // FindInCache is also expected to do an IMS check.
b07aeb1a 1453 pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
b2e465d6 1454 if (File.end() == true)
c8e572e3
MV
1455 {
1456 if (Debug == true)
1457 std::clog << "FindInCache returned end-Pointer" << std::endl;
b35d2f5f 1458 return false;
c8e572e3 1459 }
a52f938b 1460
b2e465d6 1461 Visited[File->ID] = true;
c8e572e3
MV
1462 if (Debug == true)
1463 std::clog << "with ID " << File->ID << " is valid" << std::endl;
b35d2f5f 1464 }
b07aeb1a 1465
b2e465d6
AL
1466 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1467 if (Visited[I] == false)
c8e572e3
MV
1468 {
1469 if (Debug == true)
b07aeb1a 1470 std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
b2e465d6 1471 return false;
c8e572e3 1472 }
b07aeb1a 1473
b35d2f5f
AL
1474 if (_error->PendingError() == true)
1475 {
c8e572e3
MV
1476 if (Debug == true)
1477 {
1478 std::clog << "Validity failed because of pending errors:" << std::endl;
95278287 1479 _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
c8e572e3 1480 }
b35d2f5f
AL
1481 return false;
1482 }
95278287 1483
b2e465d6 1484 if (OutMap != 0)
6c413b18 1485 *OutMap = Map.release();
f1616039
JAK
1486 if (OutCache != 0)
1487 *OutCache = CacheP.release();
b35d2f5f
AL
1488 return true;
1489}
1490 /*}}}*/
b2e465d6 1491// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 1492// ---------------------------------------------------------------------
b2e465d6
AL
1493/* Size is kind of an abstract notion that is only used for the progress
1494 meter */
b07aeb1a 1495static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
b35d2f5f 1496{
4ad8619b 1497 map_filesize_t TotalSize = 0;
b07aeb1a
DK
1498 if (List != NULL)
1499 {
1500 for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1501 {
1502 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1503 for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1504 if ((*j)->HasPackages() == true)
1505 TotalSize += (*j)->Size();
1506 }
1507 }
1508
ef74268b 1509 for (; Start < End; ++Start)
b35d2f5f 1510 {
b2e465d6 1511 if ((*Start)->HasPackages() == false)
ef74268b 1512 continue;
b2e465d6 1513 TotalSize += (*Start)->Size();
b35d2f5f 1514 }
b2e465d6 1515 return TotalSize;
2d11135a
AL
1516}
1517 /*}}}*/
b2e465d6 1518// BuildCache - Merge the list of index files into the cache /*{{{*/
b2e465d6 1519static bool BuildCache(pkgCacheGenerator &Gen,
5465192b 1520 OpProgress * const Progress,
4ad8619b 1521 map_filesize_t &CurrentSize,map_filesize_t TotalSize,
b07aeb1a 1522 pkgSourceList const * const List,
5465192b 1523 FileIterator const Start, FileIterator const End)
2d11135a 1524{
5465192b
DK
1525 bool mergeFailure = false;
1526
1527 auto const indexFileMerge = [&](pkgIndexFile * const I) {
5465192b
DK
1528 if (I->HasPackages() == false || mergeFailure)
1529 return;
1530
1531 if (I->Exists() == false)
1532 return;
1533
1534 if (I->FindInCache(Gen.GetCache()).end() == false)
1535 {
1536 _error->Warning("Duplicate sources.list entry %s",
1537 I->Describe().c_str());
1538 return;
1539 }
1540
1541 map_filesize_t const Size = I->Size();
1542 if (Progress != NULL)
1543 Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1544 CurrentSize += Size;
1545
1546 if (I->Merge(Gen,Progress) == false)
1547 mergeFailure = true;
1548 };
b07aeb1a
DK
1549
1550 if (List != NULL)
1551 {
1552 for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1553 {
3fd89e62 1554 if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
b07aeb1a
DK
1555 {
1556 _error->Warning("Duplicate sources.list entry %s",
1557 (*i)->Describe().c_str());
1558 continue;
1559 }
1560
1561 if ((*i)->Merge(Gen, Progress) == false)
1562 return false;
1563
1564 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
5465192b
DK
1565 if (Indexes != NULL)
1566 std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1567 if (mergeFailure)
1568 return false;
b07aeb1a
DK
1569 }
1570 }
1571
5465192b 1572 if (Start != End)
2d11135a 1573 {
5465192b
DK
1574 Gen.SelectReleaseFile("", "");
1575 std::for_each(Start, End, indexFileMerge);
1576 if (mergeFailure)
b2e465d6 1577 return false;
5465192b 1578 }
b35d2f5f
AL
1579 return true;
1580}
1581 /*}}}*/
5465192b
DK
1582// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
1583// ---------------------------------------------------------------------
1584/* This makes sure that the status cache (the cache that has all
1585 index files from the sources list and all local ones) is ready
1586 to be mmaped. If OutMap is not zero then a MMap object representing
1587 the cache will be stored there. This is pretty much mandetory if you
1588 are using AllowMem. AllowMem lets the function be run as non-root
1589 where it builds the cache 'fast' into a memory buffer. */
1590static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1591{
4ad8619b
DK
1592 map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1593 map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1594 map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
dcdf1ef1
DK
1595 Flags |= MMap::Moveable;
1596 if (_config->FindB("APT::Cache-Fallback", false) == true)
1597 Flags |= MMap::Fallback;
1598 if (CacheF != NULL)
1599 return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1600 else
1601 return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1602}
5465192b
DK
1603static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1604 std::string const &FileName)
1605{
1606 FileFd SCacheF(FileName, FileFd::WriteAtomic);
95278287 1607 if (SCacheF.IsOpen() == false || SCacheF.Failed())
5465192b
DK
1608 return false;
1609
1610 fchmod(SCacheF.Fd(),0644);
1611
1612 // Write out the main data
1613 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1614 return _error->Error(_("IO Error saving source cache"));
5465192b
DK
1615
1616 // Write out the proper header
1617 Gen->GetCache().HeaderP->Dirty = false;
25c7a09d 1618 Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
5465192b
DK
1619 if (SCacheF.Seek(0) == false ||
1620 SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1621 return _error->Error(_("IO Error saving source cache"));
1622 Gen->GetCache().HeaderP->Dirty = true;
5465192b
DK
1623 return true;
1624}
1625static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
6c413b18 1626 std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
5465192b 1627{
6c413b18 1628 Map.reset(CreateDynamicMMap(NULL, 0));
6789e01e
DK
1629 if (unlikely(Map->validData()) == false)
1630 return false;
5465192b 1631 FileFd CacheF(FileName, FileFd::ReadOnly);
95278287
DK
1632 if (CacheF.IsOpen() == false || CacheF.Failed())
1633 return false;
1634 _error->PushToStack();
5465192b 1635 map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
95278287
DK
1636 bool const newError = _error->PendingError();
1637 _error->MergeWithStack();
1638 if (alloc == 0 && newError)
1639 return false;
1640 if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
5465192b 1641 return false;
6c413b18 1642 Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
a133f79c 1643 return Gen->Start();
5465192b 1644}
5dd00edb 1645bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
2e5f4e45
DK
1646 MMap **OutMap, bool AllowMem)
1647 { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1648bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
c4171975 1649 MMap **OutMap,bool)
f1616039
JAK
1650{
1651 return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1652}
1653bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1654 MMap **OutMap,pkgCache **OutCache, bool)
b35d2f5f 1655{
c4171975 1656 // FIXME: deprecate the ignored AllowMem parameter
c8e572e3 1657 bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
73688d27
DK
1658
1659 std::vector<pkgIndexFile *> Files;
b2e465d6
AL
1660 if (_system->AddStatusFiles(Files) == false)
1661 return false;
c5f44afc 1662
b2e465d6 1663 // Decide if we can write to the files..
c8e572e3
MV
1664 string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1665 string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1cd1c398
DK
1666
1667 // ensure the cache directory exists
1668 if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1669 {
1670 string dir = _config->FindDir("Dir::Cache");
1671 size_t const len = dir.size();
1672 if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1673 dir = dir.substr(0, len - 5);
1674 if (CacheFile.empty() == false)
1675 CreateDirectory(dir, flNotFile(CacheFile));
1676 if (SrcCacheFile.empty() == false)
1677 CreateDirectory(dir, flNotFile(SrcCacheFile));
1678 }
1679
2e5f4e45
DK
1680 if (Progress != NULL)
1681 Progress->OverallProgress(0,1,1,_("Reading package lists"));
1682
5465192b
DK
1683 bool pkgcache_fine = false;
1684 bool srcpkgcache_fine = false;
1685 bool volatile_fine = List.GetVolatileFiles().empty();
1686
f1616039
JAK
1687 if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1688 volatile_fine ? OutCache : NULL) == true)
b2e465d6 1689 {
c8e572e3 1690 if (Debug == true)
5465192b
DK
1691 std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1692 pkgcache_fine = true;
1693 srcpkgcache_fine = true;
b2e465d6 1694 }
5465192b 1695 if (pkgcache_fine == false)
b2e465d6 1696 {
5465192b 1697 if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
41b4dee4 1698 {
41b4dee4 1699 if (Debug == true)
5465192b
DK
1700 std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1701 srcpkgcache_fine = true;
41b4dee4 1702 }
b35d2f5f 1703 }
5465192b
DK
1704
1705 if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
8ce4327b 1706 {
5465192b
DK
1707 if (Progress != NULL)
1708 Progress->OverallProgress(1,1,1,_("Reading package lists"));
1709 return true;
8ce4327b 1710 }
5465192b
DK
1711
1712 bool Writeable = false;
1713 if (srcpkgcache_fine == false || pkgcache_fine == false)
2d11135a 1714 {
5465192b
DK
1715 if (CacheFile.empty() == false)
1716 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1717 else if (SrcCacheFile.empty() == false)
1718 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1719
c8e572e3 1720 if (Debug == true)
5465192b 1721 std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
5465192b 1722 }
b2e465d6 1723
5465192b 1724 // At this point we know we need to construct something, so get storage ready
6c413b18 1725 std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
6789e01e
DK
1726 if (unlikely(Map->validData()) == false)
1727 return false;
5465192b
DK
1728 if (Debug == true)
1729 std::clog << "Open memory Map (not filebased)" << std::endl;
c8e572e3 1730
5465192b
DK
1731 std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1732 map_filesize_t CurrentSize = 0;
1733 std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1734 map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1735 if (srcpkgcache_fine == true && pkgcache_fine == false)
1736 {
1737 if (Debug == true)
1738 std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1739 if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
b2e465d6 1740 return false;
5465192b
DK
1741 srcpkgcache_fine = true;
1742 TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
b2e465d6 1743 }
5465192b 1744 else if (srcpkgcache_fine == false)
2d11135a 1745 {
c8e572e3
MV
1746 if (Debug == true)
1747 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
6c413b18 1748 Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
a133f79c
DK
1749 if (Gen->Start() == false)
1750 return false;
5465192b
DK
1751
1752 TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1753 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1754 Files.end(),Files.end()) == false)
b2e465d6 1755 return false;
5465192b 1756
b2e465d6 1757 if (Writeable == true && SrcCacheFile.empty() == false)
6c413b18 1758 if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
b2e465d6 1759 return false;
5465192b
DK
1760 }
1761
1762 if (pkgcache_fine == false)
1763 {
1764 if (Debug == true)
1765 std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1766 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1767 Files.begin(), Files.end()) == false)
b2e465d6 1768 return false;
5465192b
DK
1769
1770 if (Writeable == true && CacheFile.empty() == false)
6c413b18 1771 if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
5465192b 1772 return false;
2d11135a 1773 }
5465192b 1774
c8e572e3 1775 if (Debug == true)
a249b3e6 1776 std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
2d11135a 1777
5465192b 1778 if (volatile_fine == false)
2d11135a 1779 {
5465192b 1780 if (Gen == nullptr)
2d11135a 1781 {
5465192b
DK
1782 if (Debug == true)
1783 std::clog << "Populate new MMap with cachefile contents" << std::endl;
1784 if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1785 return false;
2d11135a 1786 }
5465192b
DK
1787
1788 Files = List.GetVolatileFiles();
1789 if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1790 Files.begin(), Files.end()) == false)
1791 return false;
2d11135a 1792 }
5465192b
DK
1793
1794 if (OutMap != nullptr)
6c413b18 1795 *OutMap = Map.release();
5465192b
DK
1796
1797 if (Debug == true)
1798 std::clog << "Everything is ready for shipping" << std::endl;
b2e465d6
AL
1799 return true;
1800}
1801 /*}}}*/
2e5f4e45 1802// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
95278287
DK
1803class APT_HIDDEN ScopedErrorMerge {
1804public:
1805 ScopedErrorMerge() { _error->PushToStack(); }
1806 ~ScopedErrorMerge() { _error->MergeWithStack(); }
1807};
5dd00edb 1808bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
2e5f4e45
DK
1809 { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1810bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
b2e465d6 1811{
73688d27 1812 std::vector<pkgIndexFile *> Files;
b2e465d6
AL
1813 if (_system->AddStatusFiles(Files) == false)
1814 return false;
dcdf1ef1 1815
95278287 1816 ScopedErrorMerge sem;
6c413b18 1817 std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
6789e01e
DK
1818 if (unlikely(Map->validData()) == false)
1819 return false;
4ad8619b
DK
1820 map_filesize_t CurrentSize = 0;
1821 map_filesize_t TotalSize = 0;
b07aeb1a 1822 TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
95278287 1823
b2e465d6 1824 // Build the status cache
2e5f4e45
DK
1825 if (Progress != NULL)
1826 Progress->OverallProgress(0,1,1,_("Reading package lists"));
6c413b18 1827 pkgCacheGenerator Gen(Map.get(),Progress);
a133f79c 1828 if (Gen.Start() == false || _error->PendingError() == true)
b2e465d6 1829 return false;
b07aeb1a
DK
1830 if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1831 Files.begin(), Files.end()) == false)
b2e465d6 1832 return false;
25396fb0 1833
b2e465d6
AL
1834 if (_error->PendingError() == true)
1835 return false;
6c413b18 1836 *OutMap = Map.release();
2d11135a 1837
b2e465d6 1838 return true;
2d11135a
AL
1839}
1840 /*}}}*/
5f4db009 1841// IsDuplicateDescription /*{{{*/
60683765 1842static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
5f4db009
DK
1843 MD5SumValue const &CurMd5, std::string const &CurLang)
1844{
22f07fc5 1845 // Descriptions in the same link-list have all the same md5
7f5aab82 1846 if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
22f07fc5
DK
1847 return false;
1848 for (; Desc.end() == false; ++Desc)
1849 if (Desc.LanguageCode() == CurLang)
5f4db009
DK
1850 return true;
1851 return false;
1852}
1853 /*}}}*/
c8a4ce6c 1854
1d3eea5c 1855pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
c9443c01 1856pkgCacheListParser::~pkgCacheListParser() {}