]> git.saurik.com Git - apt.git/blob - apt-pkg/sourcelist.cc
give a descriptive error for pipe tries with 'false'
[apt.git] / apt-pkg / sourcelist.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
4 /* ######################################################################
5
6 List of Sources
7
8 ##################################################################### */
9 /*}}}*/
10 // Include Files /*{{{*/
11 #include<config.h>
12
13 #include <apt-pkg/sourcelist.h>
14 #include <apt-pkg/cmndline.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
18 #include <apt-pkg/configuration.h>
19 #include <apt-pkg/metaindex.h>
20 #include <apt-pkg/indexfile.h>
21 #include <apt-pkg/tagfile.h>
22 #include <apt-pkg/pkgcache.h>
23 #include <apt-pkg/cacheiterators.h>
24 #include <apt-pkg/debindexfile.h>
25
26 #include <ctype.h>
27 #include <stddef.h>
28 #include <time.h>
29 #include <cstring>
30 #include <map>
31 #include <string>
32 #include <vector>
33 #include <fstream>
34 #include <algorithm>
35
36 #include <apti18n.h>
37 /*}}}*/
38
39 using namespace std;
40
41 // Global list of Items supported
42 static pkgSourceList::Type *ItmList[10];
43 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
44 unsigned long pkgSourceList::Type::GlobalListLen = 0;
45
46 // Type::Type - Constructor /*{{{*/
47 // ---------------------------------------------------------------------
48 /* Link this to the global list of items*/
49 pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
50 {
51 ItmList[GlobalListLen] = this;
52 ++GlobalListLen;
53 }
54 pkgSourceList::Type::~Type() {}
55 /*}}}*/
56 // Type::GetType - Get a specific meta for a given type /*{{{*/
57 // ---------------------------------------------------------------------
58 /* */
59 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
60 {
61 for (unsigned I = 0; I != GlobalListLen; ++I)
62 if (strcmp(GlobalList[I]->Name,Type) == 0)
63 return GlobalList[I];
64 return 0;
65 }
66 /*}}}*/
67 // Type::FixupURI - Normalize the URI and check it.. /*{{{*/
68 // ---------------------------------------------------------------------
69 /* */
70 bool pkgSourceList::Type::FixupURI(string &URI) const
71 {
72 if (URI.empty() == true)
73 return false;
74
75 if (URI.find(':') == string::npos)
76 return false;
77
78 URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
79
80 // Make sure that the URI is / postfixed
81 if (URI[URI.size() - 1] != '/')
82 URI += '/';
83
84 return true;
85 }
86 /*}}}*/
87 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, /*{{{*/
88 pkgTagSection &Tags,
89 unsigned int const i,
90 FileFd &Fd)
91 {
92 map<string, string> Options;
93
94 string Enabled = Tags.FindS("Enabled");
95 if (Enabled.empty() == false && StringToBool(Enabled) == false)
96 return true;
97
98 std::map<char const * const, std::pair<char const * const, bool> > mapping;
99 #define APT_PLUSMINUS(X, Y) \
100 mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \
101 mapping.insert(std::make_pair(X "-Add", std::make_pair(Y "+", true))); \
102 mapping.insert(std::make_pair(X "-Remove", std::make_pair(Y "-", true)))
103 APT_PLUSMINUS("Architectures", "arch");
104 APT_PLUSMINUS("Languages", "lang");
105 APT_PLUSMINUS("Targets", "target");
106 #undef APT_PLUSMINUS
107 mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false)));
108 mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false)));
109 mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false)));
110 mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
111 mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
112 mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
113 mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));
114
115 for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
116 if (Tags.Exists(m->first))
117 {
118 std::string option = Tags.FindS(m->first);
119 // for deb822 the " " is the delimiter, but the backend expects ","
120 if (m->second.second == true)
121 std::replace(option.begin(), option.end(), ' ', ',');
122 Options[m->second.first] = option;
123 }
124
125 {
126 std::string entry;
127 strprintf(entry, "%s:%i", Fd.Name().c_str(), i);
128 Options["sourceslist-entry"] = entry;
129 }
130
131 // now create one item per suite/section
132 string Suite = Tags.FindS("Suites");
133 Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
134 string const Component = Tags.FindS("Components");
135 string const URIS = Tags.FindS("URIs");
136
137 std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
138 std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
139 std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
140
141 if (list_uris.empty())
142 // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
143 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
144
145 for (std::vector<std::string>::const_iterator U = list_uris.begin();
146 U != list_uris.end(); ++U)
147 {
148 std::string URI = *U;
149 if (U->empty() || FixupURI(URI) == false)
150 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
151
152 if (list_suite.empty())
153 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
154
155 for (std::vector<std::string>::const_iterator S = list_suite.begin();
156 S != list_suite.end(); ++S)
157 {
158 if (S->empty() == false && (*S)[S->size() - 1] == '/')
159 {
160 if (list_comp.empty() == false)
161 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
162 if (CreateItem(List, URI, *S, "", Options) == false)
163 return false;
164 }
165 else
166 {
167 if (list_comp.empty())
168 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
169
170 for (std::vector<std::string>::const_iterator C = list_comp.begin();
171 C != list_comp.end(); ++C)
172 {
173 if (CreateItem(List, URI, *S, *C, Options) == false)
174 {
175 return false;
176 }
177 }
178 }
179 }
180 }
181 return true;
182 }
183 /*}}}*/
184 // Type::ParseLine - Parse a single line /*{{{*/
185 // ---------------------------------------------------------------------
186 /* This is a generic one that is the 'usual' format for sources.list
187 Weird types may override this. */
188 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
189 const char *Buffer,
190 unsigned int const CurLine,
191 string const &File) const
192 {
193 for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
194
195 // Parse option field if it exists
196 // e.g.: [ option1=value1 option2=value2 ]
197 map<string, string> Options;
198 {
199 std::string entry;
200 strprintf(entry, "%s:%i", File.c_str(), CurLine);
201 Options["sourceslist-entry"] = entry;
202 }
203 if (Buffer != 0 && Buffer[0] == '[')
204 {
205 ++Buffer; // ignore the [
206 for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
207 while (*Buffer != ']')
208 {
209 // get one option, e.g. option1=value1
210 string option;
211 if (ParseQuoteWord(Buffer,option) == false)
212 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
213
214 if (option.length() < 3)
215 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
216
217 // accept options even if the last has no space before the ]-end marker
218 if (option.at(option.length()-1) == ']')
219 {
220 for (; *Buffer != ']'; --Buffer);
221 option.resize(option.length()-1);
222 }
223
224 size_t const needle = option.find('=');
225 if (needle == string::npos)
226 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
227
228 string const key = string(option, 0, needle);
229 string const value = string(option, needle + 1, option.length());
230
231 if (key.empty() == true)
232 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
233
234 if (value.empty() == true)
235 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
236
237 Options[key] = value;
238 }
239 ++Buffer; // ignore the ]
240 for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
241 }
242
243 string URI;
244 string Dist;
245 string Section;
246
247 if (ParseQuoteWord(Buffer,URI) == false)
248 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
249 if (ParseQuoteWord(Buffer,Dist) == false)
250 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
251
252 if (FixupURI(URI) == false)
253 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
254
255 // Check for an absolute dists specification.
256 if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
257 {
258 if (ParseQuoteWord(Buffer,Section) == true)
259 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
260 Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
261 return CreateItem(List, URI, Dist, Section, Options);
262 }
263
264 // Grab the rest of the dists
265 if (ParseQuoteWord(Buffer,Section) == false)
266 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
267
268 do
269 {
270 if (CreateItem(List, URI, Dist, Section, Options) == false)
271 return false;
272 }
273 while (ParseQuoteWord(Buffer,Section) == true);
274
275 return true;
276 }
277 /*}}}*/
278 // SourceList::pkgSourceList - Constructors /*{{{*/
279 // ---------------------------------------------------------------------
280 /* */
281 pkgSourceList::pkgSourceList() : d(NULL)
282 {
283 }
284 /*}}}*/
285 // SourceList::~pkgSourceList - Destructor /*{{{*/
286 // ---------------------------------------------------------------------
287 /* */
288 pkgSourceList::~pkgSourceList()
289 {
290 for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
291 delete *I;
292 SrcList.clear();
293 for (auto F = VolatileFiles.begin(); F != VolatileFiles.end(); ++F)
294 delete (*F);
295 VolatileFiles.clear();
296 }
297 /*}}}*/
298 // SourceList::ReadMainList - Read the main source list from etc /*{{{*/
299 // ---------------------------------------------------------------------
300 /* */
301 bool pkgSourceList::ReadMainList()
302 {
303 // CNC:2003-03-03 - Multiple sources list support.
304 bool Res = true;
305 #if 0
306 Res = ReadVendors();
307 if (Res == false)
308 return false;
309 #endif
310
311 Reset();
312 // CNC:2003-11-28 - Entries in sources.list have priority over
313 // entries in sources.list.d.
314 string Main = _config->FindFile("Dir::Etc::sourcelist", "/dev/null");
315 string Parts = _config->FindDir("Dir::Etc::sourceparts", "/dev/null");
316
317 if (RealFileExists(Main) == true)
318 Res &= ReadAppend(Main);
319 else if (DirectoryExists(Parts) == false && APT::String::Endswith(Parts, "/dev/null") == false)
320 // Only warn if there are no sources.list.d.
321 _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
322
323 if (DirectoryExists(Parts) == true)
324 Res &= ReadSourceDir(Parts);
325 else if (Main.empty() == false && RealFileExists(Main) == false &&
326 APT::String::Endswith(Parts, "/dev/null") == false)
327 // Only warn if there is no sources.list file.
328 _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
329
330 return Res;
331 }
332 /*}}}*/
333 // SourceList::Reset - Clear the sourcelist contents /*{{{*/
334 // ---------------------------------------------------------------------
335 /* */
336 void pkgSourceList::Reset()
337 {
338 for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
339 delete *I;
340 SrcList.clear();
341 }
342 /*}}}*/
343 // SourceList::Read - Parse the sourcelist file /*{{{*/
344 // ---------------------------------------------------------------------
345 /* */
346 bool pkgSourceList::Read(string const &File)
347 {
348 Reset();
349 return ReadAppend(File);
350 }
351 /*}}}*/
352 // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/
353 // ---------------------------------------------------------------------
354 /* */
355 bool pkgSourceList::ReadAppend(string const &File)
356 {
357 if (flExtension(File) == "sources")
358 return ParseFileDeb822(File);
359 else
360 return ParseFileOldStyle(File);
361 }
362 /*}}}*/
363 // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/
364 // ---------------------------------------------------------------------
365 /* */
366 bool pkgSourceList::ParseFileOldStyle(std::string const &File)
367 {
368 // Open the stream for reading
369 ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
370 if (F.fail() == true)
371 return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
372
373 std::string Buffer;
374 for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
375 {
376 // remove comments
377 size_t curpos = 0;
378 while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
379 {
380 size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
381 size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
382 if (openbrackets > closedbrackets)
383 {
384 // a # in an option, unlikely, but oh well, it was supported so stick to it
385 ++curpos;
386 continue;
387 }
388 Buffer.erase(curpos);
389 break;
390 }
391 // remove spaces before/after
392 curpos = Buffer.find_first_not_of(" \t\r");
393 if (curpos != 0)
394 Buffer.erase(0, curpos);
395 curpos = Buffer.find_last_not_of(" \t\r");
396 if (curpos != std::string::npos)
397 Buffer.erase(curpos + 1);
398
399 if (Buffer.empty())
400 continue;
401
402 // Grok it
403 std::string const LineType = Buffer.substr(0, Buffer.find_first_of(" \t\v"));
404 if (LineType.empty() || LineType == Buffer)
405 return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
406
407 Type *Parse = Type::GetType(LineType.c_str());
408 if (Parse == 0)
409 return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
410
411 if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
412 return false;
413 }
414 return true;
415 }
416 /*}}}*/
417 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/
418 // ---------------------------------------------------------------------
419 /* Returns: the number of stanzas parsed*/
420 bool pkgSourceList::ParseFileDeb822(string const &File)
421 {
422 unsigned int i = 1;
423
424 // see if we can read the file
425 FileFd Fd(File, FileFd::ReadOnly);
426 pkgTagFile Sources(&Fd, pkgTagFile::SUPPORT_COMMENTS);
427 if (Fd.IsOpen() == false || Fd.Failed())
428 return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
429
430 // read step by step
431 pkgTagSection Tags;
432 while (Sources.Step(Tags) == true)
433 {
434 if(Tags.Exists("Types") == false)
435 return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
436
437 string const types = Tags.FindS("Types");
438 std::vector<std::string> const list_types = VectorizeString(types, ' ');
439 for (std::vector<std::string>::const_iterator I = list_types.begin();
440 I != list_types.end(); ++I)
441 {
442 Type *Parse = Type::GetType((*I).c_str());
443 if (Parse == 0)
444 {
445 _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
446 return false;
447 }
448
449 if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
450 return false;
451
452 ++i;
453 }
454 }
455 return true;
456 }
457 /*}}}*/
458 // SourceList::FindIndex - Get the index associated with a file /*{{{*/
459 static bool FindInIndexFileContainer(std::vector<pkgIndexFile *> const &Cont, pkgCache::PkgFileIterator const &File, pkgIndexFile *&Found)
460 {
461 auto const J = std::find_if(Cont.begin(), Cont.end(), [&File](pkgIndexFile const * const J) {
462 return J->FindInCache(*File.Cache()) == File;
463 });
464 if (J != Cont.end())
465 {
466 Found = (*J);
467 return true;
468 }
469 return false;
470 }
471 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
472 pkgIndexFile *&Found) const
473 {
474 for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
475 if (FindInIndexFileContainer(*(*I)->GetIndexFiles(), File, Found))
476 return true;
477
478 return FindInIndexFileContainer(VolatileFiles, File, Found);
479 }
480 /*}}}*/
481 // SourceList::GetIndexes - Load the index files into the downloader /*{{{*/
482 // ---------------------------------------------------------------------
483 /* */
484 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
485 {
486 for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
487 if ((*I)->GetIndexes(Owner,GetAll) == false)
488 return false;
489 return true;
490 }
491 /*}}}*/
492 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
493 // SourceList::ReadSourceDir - Read a directory with sources files
494 // Based on ReadConfigDir() /*{{{*/
495 // ---------------------------------------------------------------------
496 /* */
497 bool pkgSourceList::ReadSourceDir(string const &Dir)
498 {
499 std::vector<std::string> ext;
500 ext.push_back("list");
501 ext.push_back("sources");
502 std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
503
504 // Read the files
505 for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
506 if (ReadAppend(*I) == false)
507 return false;
508 return true;
509
510 }
511 /*}}}*/
512 // GetLastModified() /*{{{*/
513 // ---------------------------------------------------------------------
514 /* */
515 time_t pkgSourceList::GetLastModifiedTime()
516 {
517 vector<string> List;
518
519 string Main = _config->FindFile("Dir::Etc::sourcelist");
520 string Parts = _config->FindDir("Dir::Etc::sourceparts");
521
522 // go over the parts
523 if (DirectoryExists(Parts) == true)
524 List = GetListOfFilesInDir(Parts, "list", true);
525
526 // calculate the time
527 std::vector<time_t> modtimes;
528 modtimes.reserve(1 + List.size());
529 modtimes.push_back(GetModificationTime(Main));
530 std::transform(List.begin(), List.end(), std::back_inserter(modtimes), GetModificationTime);
531 auto const maxmtime = std::max_element(modtimes.begin(), modtimes.end());
532 return *maxmtime;
533 }
534 /*}}}*/
535 std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const /*{{{*/
536 {
537 return VolatileFiles;
538 }
539 /*}}}*/
540 void pkgSourceList::AddVolatileFile(pkgIndexFile * const File) /*{{{*/
541 {
542 if (File != nullptr)
543 VolatileFiles.push_back(File);
544 }
545 /*}}}*/
546 bool pkgSourceList::AddVolatileFile(std::string const &File) /*{{{*/
547 {
548 // Note: FileExists matches directories and links, too!
549 if (File.empty() || FileExists(File) == false)
550 return false;
551
552 std::string const ext = flExtension(File);
553 if (ext == "deb")
554 AddVolatileFile(new debDebPkgFileIndex(File));
555 else if (ext == "dsc")
556 AddVolatileFile(new debDscFileIndex(File));
557 else if (FileExists(flCombine(File, "debian/control")))
558 AddVolatileFile(new debDscFileIndex(flCombine(File, "debian/control")));
559 else
560 return false;
561
562 return true;
563 }
564 /*}}}*/
565 void pkgSourceList::AddVolatileFiles(CommandLine &CmdL, std::vector<const char*> * const VolatileCmdL)/*{{{*/
566 {
567 std::remove_if(CmdL.FileList + 1, CmdL.FileList + 1 + CmdL.FileSize(), [&](char const * const I) {
568 if (I != nullptr && (I[0] == '/' || (I[0] == '.' && ((I[1] == '.' && I[2] == '/') || I[1] == '/'))))
569 {
570 if (AddVolatileFile(I))
571 {
572 if (VolatileCmdL != nullptr)
573 VolatileCmdL->push_back(I);
574 }
575 else
576 _error->Error(_("Unsupported file %s given on commandline"), I);
577 return true;
578 }
579 return false;
580 });
581 }
582 /*}}}*/