]> git.saurik.com Git - apt.git/blob - apt-pkg/sourcelist.cc
test framework: Unset http proxy variables in setupenvironment
[apt.git] / apt-pkg / sourcelist.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
4 /* ######################################################################
5
6 List of Sources
7
8 ##################################################################### */
9 /*}}}*/
10 // Include Files /*{{{*/
11 #include<config.h>
12
13 #include <apt-pkg/sourcelist.h>
14 #include <apt-pkg/cmndline.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
18 #include <apt-pkg/configuration.h>
19 #include <apt-pkg/metaindex.h>
20 #include <apt-pkg/indexfile.h>
21 #include <apt-pkg/tagfile.h>
22 #include <apt-pkg/pkgcache.h>
23 #include <apt-pkg/cacheiterators.h>
24 #include <apt-pkg/debindexfile.h>
25
26 #include <ctype.h>
27 #include <stddef.h>
28 #include <time.h>
29 #include <cstring>
30 #include <map>
31 #include <string>
32 #include <vector>
33 #include <fstream>
34 #include <algorithm>
35
36 #include <apti18n.h>
37 /*}}}*/
38
39 using namespace std;
40
41 // Global list of Items supported
42 static pkgSourceList::Type *ItmList[10];
43 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
44 unsigned long pkgSourceList::Type::GlobalListLen = 0;
45
46 // Type::Type - Constructor /*{{{*/
47 // ---------------------------------------------------------------------
48 /* Link this to the global list of items*/
49 pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
50 {
51 ItmList[GlobalListLen] = this;
52 ++GlobalListLen;
53 }
54 pkgSourceList::Type::~Type() {}
55 /*}}}*/
56 // Type::GetType - Get a specific meta for a given type /*{{{*/
57 // ---------------------------------------------------------------------
58 /* */
59 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
60 {
61 for (unsigned I = 0; I != GlobalListLen; ++I)
62 if (strcmp(GlobalList[I]->Name,Type) == 0)
63 return GlobalList[I];
64 return 0;
65 }
66 /*}}}*/
67 // Type::FixupURI - Normalize the URI and check it.. /*{{{*/
68 // ---------------------------------------------------------------------
69 /* */
70 bool pkgSourceList::Type::FixupURI(string &URI) const
71 {
72 if (URI.empty() == true)
73 return false;
74
75 if (URI.find(':') == string::npos)
76 return false;
77
78 URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
79
80 // Make sure that the URI is / postfixed
81 if (URI[URI.size() - 1] != '/')
82 URI += '/';
83
84 return true;
85 }
86 /*}}}*/
87 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, /*{{{*/
88 pkgTagSection &Tags,
89 unsigned int const i,
90 FileFd &Fd)
91 {
92 map<string, string> Options;
93
94 string Enabled = Tags.FindS("Enabled");
95 if (Enabled.empty() == false && StringToBool(Enabled) == false)
96 return true;
97
98 std::map<char const * const, std::pair<char const * const, bool> > mapping;
99 #define APT_PLUSMINUS(X, Y) \
100 mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \
101 mapping.insert(std::make_pair(X "Add", std::make_pair(Y "+", true))); \
102 mapping.insert(std::make_pair(X "Remove", std::make_pair(Y "-", true)))
103 APT_PLUSMINUS("Architectures", "arch");
104 APT_PLUSMINUS("Languages", "lang");
105 APT_PLUSMINUS("Targets", "target");
106 #undef APT_PLUSMINUS
107 mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false)));
108 mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false)));
109 mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false)));
110 mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
111 mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
112 mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
113 mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));
114
115 for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
116 if (Tags.Exists(m->first))
117 {
118 std::string option = Tags.FindS(m->first);
119 // for deb822 the " " is the delimiter, but the backend expects ","
120 if (m->second.second == true)
121 std::replace(option.begin(), option.end(), ' ', ',');
122 Options[m->second.first] = option;
123 }
124
125 {
126 std::string entry;
127 strprintf(entry, "%s:%i", Fd.Name().c_str(), i);
128 Options["sourceslist-entry"] = entry;
129 }
130
131 // now create one item per suite/section
132 string Suite = Tags.FindS("Suites");
133 Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
134 string const Component = Tags.FindS("Components");
135 string const URIS = Tags.FindS("URIs");
136
137 std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
138 std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
139 std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
140
141 if (list_uris.empty())
142 // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
143 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
144
145 for (std::vector<std::string>::const_iterator U = list_uris.begin();
146 U != list_uris.end(); ++U)
147 {
148 std::string URI = *U;
149 if (U->empty() || FixupURI(URI) == false)
150 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
151
152 if (list_suite.empty())
153 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
154
155 for (std::vector<std::string>::const_iterator S = list_suite.begin();
156 S != list_suite.end(); ++S)
157 {
158 if (S->empty() == false && (*S)[S->size() - 1] == '/')
159 {
160 if (list_comp.empty() == false)
161 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
162 if (CreateItem(List, URI, *S, "", Options) == false)
163 return false;
164 }
165 else
166 {
167 if (list_comp.empty())
168 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
169
170 for (std::vector<std::string>::const_iterator C = list_comp.begin();
171 C != list_comp.end(); ++C)
172 {
173 if (CreateItem(List, URI, *S, *C, Options) == false)
174 {
175 return false;
176 }
177 }
178 }
179 }
180 }
181 return true;
182 }
183 /*}}}*/
184 // Type::ParseLine - Parse a single line /*{{{*/
185 // ---------------------------------------------------------------------
186 /* This is a generic one that is the 'usual' format for sources.list
187 Weird types may override this. */
188 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
189 const char *Buffer,
190 unsigned int const CurLine,
191 string const &File) const
192 {
193 for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
194
195 // Parse option field if it exists
196 // e.g.: [ option1=value1 option2=value2 ]
197 map<string, string> Options;
198 {
199 std::string entry;
200 strprintf(entry, "%s:%i", File.c_str(), CurLine);
201 Options["sourceslist-entry"] = entry;
202 }
203 if (Buffer != 0 && Buffer[0] == '[')
204 {
205 ++Buffer; // ignore the [
206 for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
207 while (*Buffer != ']')
208 {
209 // get one option, e.g. option1=value1
210 string option;
211 if (ParseQuoteWord(Buffer,option) == false)
212 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
213
214 if (option.length() < 3)
215 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
216
217 // accept options even if the last has no space before the ]-end marker
218 if (option.at(option.length()-1) == ']')
219 {
220 for (; *Buffer != ']'; --Buffer);
221 option.resize(option.length()-1);
222 }
223
224 size_t const needle = option.find('=');
225 if (needle == string::npos)
226 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
227
228 string const key = string(option, 0, needle);
229 string const value = string(option, needle + 1, option.length());
230
231 if (key.empty() == true)
232 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
233
234 if (value.empty() == true)
235 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
236
237 Options[key] = value;
238 }
239 ++Buffer; // ignore the ]
240 for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
241 }
242
243 string URI;
244 string Dist;
245 string Section;
246
247 if (ParseQuoteWord(Buffer,URI) == false)
248 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
249 if (ParseQuoteWord(Buffer,Dist) == false)
250 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
251
252 if (FixupURI(URI) == false)
253 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
254
255 // Check for an absolute dists specification.
256 if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
257 {
258 if (ParseQuoteWord(Buffer,Section) == true)
259 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
260 Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
261 return CreateItem(List, URI, Dist, Section, Options);
262 }
263
264 // Grab the rest of the dists
265 if (ParseQuoteWord(Buffer,Section) == false)
266 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
267
268 do
269 {
270 if (CreateItem(List, URI, Dist, Section, Options) == false)
271 return false;
272 }
273 while (ParseQuoteWord(Buffer,Section) == true);
274
275 return true;
276 }
277 /*}}}*/
278 // SourceList::pkgSourceList - Constructors /*{{{*/
279 // ---------------------------------------------------------------------
280 /* */
281 pkgSourceList::pkgSourceList() : d(NULL)
282 {
283 }
284 /*}}}*/
285 // SourceList::~pkgSourceList - Destructor /*{{{*/
286 // ---------------------------------------------------------------------
287 /* */
288 pkgSourceList::~pkgSourceList()
289 {
290 for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
291 delete *I;
292 SrcList.clear();
293 for (auto F = VolatileFiles.begin(); F != VolatileFiles.end(); ++F)
294 delete (*F);
295 VolatileFiles.clear();
296 }
297 /*}}}*/
298 // SourceList::ReadMainList - Read the main source list from etc /*{{{*/
299 // ---------------------------------------------------------------------
300 /* */
301 bool pkgSourceList::ReadMainList()
302 {
303 // CNC:2003-03-03 - Multiple sources list support.
304 bool Res = true;
305 #if 0
306 Res = ReadVendors();
307 if (Res == false)
308 return false;
309 #endif
310
311 Reset();
312 // CNC:2003-11-28 - Entries in sources.list have priority over
313 // entries in sources.list.d.
314 string Main = _config->FindFile("Dir::Etc::sourcelist");
315 string Parts = _config->FindDir("Dir::Etc::sourceparts");
316
317 if (RealFileExists(Main) == true)
318 Res &= ReadAppend(Main);
319 else if (DirectoryExists(Parts) == false)
320 // Only warn if there are no sources.list.d.
321 _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
322
323 if (DirectoryExists(Parts) == true)
324 Res &= ReadSourceDir(Parts);
325 else if (RealFileExists(Main) == false)
326 // Only warn if there is no sources.list file.
327 _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
328
329 return Res;
330 }
331 /*}}}*/
332 // SourceList::Reset - Clear the sourcelist contents /*{{{*/
333 // ---------------------------------------------------------------------
334 /* */
335 void pkgSourceList::Reset()
336 {
337 for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
338 delete *I;
339 SrcList.clear();
340 }
341 /*}}}*/
342 // SourceList::Read - Parse the sourcelist file /*{{{*/
343 // ---------------------------------------------------------------------
344 /* */
345 bool pkgSourceList::Read(string const &File)
346 {
347 Reset();
348 return ReadAppend(File);
349 }
350 /*}}}*/
351 // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/
352 // ---------------------------------------------------------------------
353 /* */
354 bool pkgSourceList::ReadAppend(string const &File)
355 {
356 if (flExtension(File) == "sources")
357 return ParseFileDeb822(File);
358 else
359 return ParseFileOldStyle(File);
360 }
361 /*}}}*/
362 // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/
363 // ---------------------------------------------------------------------
364 /* */
365 bool pkgSourceList::ParseFileOldStyle(std::string const &File)
366 {
367 // Open the stream for reading
368 ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
369 if (F.fail() == true)
370 return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
371
372 std::string Buffer;
373 for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
374 {
375 // remove comments
376 size_t curpos = 0;
377 while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
378 {
379 size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
380 size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
381 if (openbrackets > closedbrackets)
382 {
383 // a # in an option, unlikely, but oh well, it was supported so stick to it
384 ++curpos;
385 continue;
386 }
387 Buffer.erase(curpos);
388 break;
389 }
390 // remove spaces before/after
391 curpos = Buffer.find_first_not_of(" \t\r");
392 if (curpos != 0)
393 Buffer.erase(0, curpos);
394 curpos = Buffer.find_last_not_of(" \t\r");
395 if (curpos != std::string::npos)
396 Buffer.erase(curpos + 1);
397
398 if (Buffer.empty())
399 continue;
400
401 // Grok it
402 std::string const LineType = Buffer.substr(0, Buffer.find_first_of(" \t\v"));
403 if (LineType.empty() || LineType == Buffer)
404 return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
405
406 Type *Parse = Type::GetType(LineType.c_str());
407 if (Parse == 0)
408 return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
409
410 if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
411 return false;
412 }
413 return true;
414 }
415 /*}}}*/
416 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/
417 // ---------------------------------------------------------------------
418 /* Returns: the number of stanzas parsed*/
419 bool pkgSourceList::ParseFileDeb822(string const &File)
420 {
421 pkgUserTagSection Tags;
422 unsigned int i = 1;
423
424 // see if we can read the file
425 FileFd Fd(File, FileFd::ReadOnly);
426 pkgTagFile Sources(&Fd);
427 if (Fd.IsOpen() == false || Fd.Failed())
428 return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
429
430 // read step by step
431 while (Sources.Step(Tags) == true)
432 {
433 if(Tags.Exists("Types") == false)
434 return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
435
436 string const types = Tags.FindS("Types");
437 std::vector<std::string> const list_types = VectorizeString(types, ' ');
438 for (std::vector<std::string>::const_iterator I = list_types.begin();
439 I != list_types.end(); ++I)
440 {
441 Type *Parse = Type::GetType((*I).c_str());
442 if (Parse == 0)
443 {
444 _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
445 return false;
446 }
447
448 if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
449 return false;
450
451 ++i;
452 }
453 }
454 return true;
455 }
456 /*}}}*/
457 // SourceList::FindIndex - Get the index associated with a file /*{{{*/
458 static bool FindInIndexFileContainer(std::vector<pkgIndexFile *> const &Cont, pkgCache::PkgFileIterator const &File, pkgIndexFile *&Found)
459 {
460 auto const J = std::find_if(Cont.begin(), Cont.end(), [&File](pkgIndexFile const * const J) {
461 return J->FindInCache(*File.Cache()) == File;
462 });
463 if (J != Cont.end())
464 {
465 Found = (*J);
466 return true;
467 }
468 return false;
469 }
470 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
471 pkgIndexFile *&Found) const
472 {
473 for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
474 if (FindInIndexFileContainer(*(*I)->GetIndexFiles(), File, Found))
475 return true;
476
477 return FindInIndexFileContainer(VolatileFiles, File, Found);
478 }
479 /*}}}*/
480 // SourceList::GetIndexes - Load the index files into the downloader /*{{{*/
481 // ---------------------------------------------------------------------
482 /* */
483 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
484 {
485 for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
486 if ((*I)->GetIndexes(Owner,GetAll) == false)
487 return false;
488 return true;
489 }
490 /*}}}*/
491 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
492 // SourceList::ReadSourceDir - Read a directory with sources files
493 // Based on ReadConfigDir() /*{{{*/
494 // ---------------------------------------------------------------------
495 /* */
496 bool pkgSourceList::ReadSourceDir(string const &Dir)
497 {
498 std::vector<std::string> ext;
499 ext.push_back("list");
500 ext.push_back("sources");
501 std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
502
503 // Read the files
504 for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
505 if (ReadAppend(*I) == false)
506 return false;
507 return true;
508
509 }
510 /*}}}*/
511 // GetLastModified() /*{{{*/
512 // ---------------------------------------------------------------------
513 /* */
514 time_t pkgSourceList::GetLastModifiedTime()
515 {
516 vector<string> List;
517
518 string Main = _config->FindFile("Dir::Etc::sourcelist");
519 string Parts = _config->FindDir("Dir::Etc::sourceparts");
520
521 // go over the parts
522 if (DirectoryExists(Parts) == true)
523 List = GetListOfFilesInDir(Parts, "list", true);
524
525 // calculate the time
526 std::vector<time_t> modtimes;
527 modtimes.reserve(1 + List.size());
528 modtimes.push_back(GetModificationTime(Main));
529 std::transform(List.begin(), List.end(), std::back_inserter(modtimes), GetModificationTime);
530 auto const maxmtime = std::max_element(modtimes.begin(), modtimes.end());
531 return *maxmtime;
532 }
533 /*}}}*/
534 std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const /*{{{*/
535 {
536 return VolatileFiles;
537 }
538 /*}}}*/
539 void pkgSourceList::AddVolatileFile(pkgIndexFile * const File) /*{{{*/
540 {
541 if (File != nullptr)
542 VolatileFiles.push_back(File);
543 }
544 /*}}}*/
545 bool pkgSourceList::AddVolatileFile(std::string const &File) /*{{{*/
546 {
547 // Note: FileExists matches directories and links, too!
548 if (File.empty() || FileExists(File) == false)
549 return false;
550
551 std::string const ext = flExtension(File);
552 if (ext == "deb")
553 AddVolatileFile(new debDebPkgFileIndex(File));
554 else if (ext == "dsc")
555 AddVolatileFile(new debDscFileIndex(File));
556 else if (FileExists(flCombine(File, "debian/control")))
557 AddVolatileFile(new debDscFileIndex(flCombine(File, "debian/control")));
558 else
559 return false;
560
561 return true;
562 }
563 /*}}}*/
564 void pkgSourceList::AddVolatileFiles(CommandLine &CmdL, std::vector<const char*> * const VolatileCmdL)/*{{{*/
565 {
566 std::remove_if(CmdL.FileList + 1, CmdL.FileList + 1 + CmdL.FileSize(), [&](char const * const I) {
567 if (I != nullptr && (I[0] == '/' || (I[0] == '.' && ((I[1] == '.' && I[2] == '/') || I[1] == '/'))))
568 {
569 if (AddVolatileFile(I))
570 {
571 if (VolatileCmdL != nullptr)
572 VolatileCmdL->push_back(I);
573 }
574 else
575 _error->Error(_("Unsupported file %s given on commandline"), I);
576 return true;
577 }
578 return false;
579 });
580 }
581 /*}}}*/