1 // -*- mode: cpp; mode: fold -*-
3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
4 /* ######################################################################
8 ##################################################################### */
10 // Include Files /*{{{*/
13 #include <apt-pkg/sourcelist.h>
14 #include <apt-pkg/cmndline.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
18 #include <apt-pkg/configuration.h>
19 #include <apt-pkg/metaindex.h>
20 #include <apt-pkg/indexfile.h>
21 #include <apt-pkg/tagfile.h>
22 #include <apt-pkg/pkgcache.h>
23 #include <apt-pkg/cacheiterators.h>
24 #include <apt-pkg/debindexfile.h>
25 #include <apt-pkg/debsrcrecords.h>
42 // Global list of Items supported
43 static pkgSourceList::Type
*ItmList
[10];
44 pkgSourceList::Type
**pkgSourceList::Type::GlobalList
= ItmList
;
45 unsigned long pkgSourceList::Type::GlobalListLen
= 0;
47 // Type::Type - Constructor /*{{{*/
48 // ---------------------------------------------------------------------
49 /* Link this to the global list of items*/
50 pkgSourceList::Type::Type(char const * const pName
, char const * const pLabel
) : Name(pName
), Label(pLabel
)
52 ItmList
[GlobalListLen
] = this;
55 pkgSourceList::Type::~Type() {}
57 // Type::GetType - Get a specific meta for a given type /*{{{*/
58 // ---------------------------------------------------------------------
60 pkgSourceList::Type
*pkgSourceList::Type::GetType(const char *Type
)
62 for (unsigned I
= 0; I
!= GlobalListLen
; ++I
)
63 if (strcmp(GlobalList
[I
]->Name
,Type
) == 0)
68 // Type::FixupURI - Normalize the URI and check it.. /*{{{*/
69 // ---------------------------------------------------------------------
71 bool pkgSourceList::Type::FixupURI(string
&URI
) const
73 if (URI
.empty() == true)
76 if (URI
.find(':') == string::npos
)
79 URI
= SubstVar(URI
,"$(ARCH)",_config
->Find("APT::Architecture"));
81 // Make sure that the URI is / postfixed
82 if (URI
[URI
.size() - 1] != '/')
88 bool pkgSourceList::Type::ParseStanza(vector
<metaIndex
*> &List
, /*{{{*/
93 map
<string
, string
> Options
;
95 string Enabled
= Tags
.FindS("Enabled");
96 if (Enabled
.empty() == false && StringToBool(Enabled
) == false)
99 std::map
<char const * const, std::pair
<char const * const, bool> > mapping
;
100 #define APT_PLUSMINUS(X, Y) \
101 mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \
102 mapping.insert(std::make_pair(X "-Add", std::make_pair(Y "+", true))); \
103 mapping.insert(std::make_pair(X "-Remove", std::make_pair(Y "-", true)))
104 APT_PLUSMINUS("Architectures", "arch");
105 APT_PLUSMINUS("Languages", "lang");
106 APT_PLUSMINUS("Targets", "target");
108 mapping
.insert(std::make_pair("Trusted", std::make_pair("trusted", false)));
109 mapping
.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false)));
110 mapping
.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false)));
111 mapping
.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
112 mapping
.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
113 mapping
.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
114 mapping
.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));
116 for (std::map
<char const * const, std::pair
<char const * const, bool> >::const_iterator m
= mapping
.begin(); m
!= mapping
.end(); ++m
)
117 if (Tags
.Exists(m
->first
))
119 std::string option
= Tags
.FindS(m
->first
);
120 // for deb822 the " " is the delimiter, but the backend expects ","
121 if (m
->second
.second
== true)
122 std::replace(option
.begin(), option
.end(), ' ', ',');
123 Options
[m
->second
.first
] = option
;
128 strprintf(entry
, "%s:%i", Fd
.Name().c_str(), i
);
129 Options
["sourceslist-entry"] = entry
;
132 // now create one item per suite/section
133 string Suite
= Tags
.FindS("Suites");
134 Suite
= SubstVar(Suite
,"$(ARCH)",_config
->Find("APT::Architecture"));
135 string
const Component
= Tags
.FindS("Components");
136 string
const URIS
= Tags
.FindS("URIs");
138 std::vector
<std::string
> const list_uris
= VectorizeString(URIS
, ' ');
139 std::vector
<std::string
> const list_suite
= VectorizeString(Suite
, ' ');
140 std::vector
<std::string
> const list_comp
= VectorizeString(Component
, ' ');
142 if (list_uris
.empty())
143 // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
144 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "URI");
146 for (std::vector
<std::string
>::const_iterator U
= list_uris
.begin();
147 U
!= list_uris
.end(); ++U
)
149 std::string URI
= *U
;
150 if (U
->empty() || FixupURI(URI
) == false)
151 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "URI parse");
153 if (list_suite
.empty())
154 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "Suite");
156 for (std::vector
<std::string
>::const_iterator S
= list_suite
.begin();
157 S
!= list_suite
.end(); ++S
)
159 if (S
->empty() == false && (*S
)[S
->size() - 1] == '/')
161 if (list_comp
.empty() == false)
162 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "absolute Suite Component");
163 if (CreateItem(List
, URI
, *S
, "", Options
) == false)
168 if (list_comp
.empty())
169 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "Component");
171 for (std::vector
<std::string
>::const_iterator C
= list_comp
.begin();
172 C
!= list_comp
.end(); ++C
)
174 if (CreateItem(List
, URI
, *S
, *C
, Options
) == false)
185 // Type::ParseLine - Parse a single line /*{{{*/
186 // ---------------------------------------------------------------------
187 /* This is a generic one that is the 'usual' format for sources.list
188 Weird types may override this. */
189 bool pkgSourceList::Type::ParseLine(vector
<metaIndex
*> &List
,
191 unsigned int const CurLine
,
192 string
const &File
) const
194 for (;Buffer
!= 0 && isspace(*Buffer
); ++Buffer
); // Skip whitespaces
196 // Parse option field if it exists
197 // e.g.: [ option1=value1 option2=value2 ]
198 map
<string
, string
> Options
;
201 strprintf(entry
, "%s:%i", File
.c_str(), CurLine
);
202 Options
["sourceslist-entry"] = entry
;
204 if (Buffer
!= 0 && Buffer
[0] == '[')
206 ++Buffer
; // ignore the [
207 for (;Buffer
!= 0 && isspace(*Buffer
); ++Buffer
); // Skip whitespaces
208 while (*Buffer
!= ']')
210 // get one option, e.g. option1=value1
212 if (ParseQuoteWord(Buffer
,option
) == false)
213 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] unparseable");
215 if (option
.length() < 3)
216 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] too short");
218 // accept options even if the last has no space before the ]-end marker
219 if (option
.at(option
.length()-1) == ']')
221 for (; *Buffer
!= ']'; --Buffer
);
222 option
.resize(option
.length()-1);
225 size_t const needle
= option
.find('=');
226 if (needle
== string::npos
)
227 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] not assignment");
229 string
const key
= string(option
, 0, needle
);
230 string
const value
= string(option
, needle
+ 1, option
.length());
232 if (key
.empty() == true)
233 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] no key");
235 if (value
.empty() == true)
236 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] no value");
238 Options
[key
] = value
;
240 ++Buffer
; // ignore the ]
241 for (;Buffer
!= 0 && isspace(*Buffer
); ++Buffer
); // Skip whitespaces
248 if (ParseQuoteWord(Buffer
,URI
) == false)
249 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "URI");
250 if (ParseQuoteWord(Buffer
,Dist
) == false)
251 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "Suite");
253 if (FixupURI(URI
) == false)
254 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "URI parse");
256 // Check for an absolute dists specification.
257 if (Dist
.empty() == false && Dist
[Dist
.size() - 1] == '/')
259 if (ParseQuoteWord(Buffer
,Section
) == true)
260 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "absolute Suite Component");
261 Dist
= SubstVar(Dist
,"$(ARCH)",_config
->Find("APT::Architecture"));
262 return CreateItem(List
, URI
, Dist
, Section
, Options
);
265 // Grab the rest of the dists
266 if (ParseQuoteWord(Buffer
,Section
) == false)
267 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "Component");
271 if (CreateItem(List
, URI
, Dist
, Section
, Options
) == false)
274 while (ParseQuoteWord(Buffer
,Section
) == true);
279 // SourceList::pkgSourceList - Constructors /*{{{*/
280 // ---------------------------------------------------------------------
282 pkgSourceList::pkgSourceList() : d(NULL
)
286 // SourceList::~pkgSourceList - Destructor /*{{{*/
287 // ---------------------------------------------------------------------
289 pkgSourceList::~pkgSourceList()
291 for (const_iterator I
= SrcList
.begin(); I
!= SrcList
.end(); ++I
)
294 for (auto F
= VolatileFiles
.begin(); F
!= VolatileFiles
.end(); ++F
)
296 VolatileFiles
.clear();
299 // SourceList::ReadMainList - Read the main source list from etc /*{{{*/
300 // ---------------------------------------------------------------------
302 bool pkgSourceList::ReadMainList()
304 // CNC:2003-03-03 - Multiple sources list support.
313 // CNC:2003-11-28 - Entries in sources.list have priority over
314 // entries in sources.list.d.
315 string Main
= _config
->FindFile("Dir::Etc::sourcelist", "/dev/null");
316 string Parts
= _config
->FindDir("Dir::Etc::sourceparts", "/dev/null");
318 if (RealFileExists(Main
) == true)
319 Res
&= ReadAppend(Main
);
320 else if (DirectoryExists(Parts
) == false && APT::String::Endswith(Parts
, "/dev/null") == false)
321 // Only warn if there are no sources.list.d.
322 _error
->WarningE("DirectoryExists", _("Unable to read %s"), Parts
.c_str());
324 if (DirectoryExists(Parts
) == true)
325 Res
&= ReadSourceDir(Parts
);
326 else if (Main
.empty() == false && RealFileExists(Main
) == false &&
327 APT::String::Endswith(Parts
, "/dev/null") == false)
328 // Only warn if there is no sources.list file.
329 _error
->WarningE("RealFileExists", _("Unable to read %s"), Main
.c_str());
331 for (auto && file
: _config
->FindVector("APT::Sources::With"))
332 Res
&= AddVolatileFile(file
, nullptr);
337 // SourceList::Reset - Clear the sourcelist contents /*{{{*/
338 // ---------------------------------------------------------------------
340 void pkgSourceList::Reset()
342 for (const_iterator I
= SrcList
.begin(); I
!= SrcList
.end(); ++I
)
347 // SourceList::Read - Parse the sourcelist file /*{{{*/
348 // ---------------------------------------------------------------------
350 bool pkgSourceList::Read(string
const &File
)
353 return ReadAppend(File
);
356 // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/
357 // ---------------------------------------------------------------------
359 bool pkgSourceList::ReadAppend(string
const &File
)
361 if (flExtension(File
) == "sources")
362 return ParseFileDeb822(File
);
364 return ParseFileOldStyle(File
);
367 // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/
368 // ---------------------------------------------------------------------
370 bool pkgSourceList::ParseFileOldStyle(std::string
const &File
)
372 // Open the stream for reading
373 ifstream
F(File
.c_str(),ios::in
/*| ios::nocreate*/);
374 if (F
.fail() == true)
375 return _error
->Errno("ifstream::ifstream",_("Opening %s"),File
.c_str());
378 for (unsigned int CurLine
= 1; std::getline(F
, Buffer
); ++CurLine
)
382 while ((curpos
= Buffer
.find('#', curpos
)) != std::string::npos
)
384 size_t const openbrackets
= std::count(Buffer
.begin(), Buffer
.begin() + curpos
, '[');
385 size_t const closedbrackets
= std::count(Buffer
.begin(), Buffer
.begin() + curpos
, ']');
386 if (openbrackets
> closedbrackets
)
388 // a # in an option, unlikely, but oh well, it was supported so stick to it
392 Buffer
.erase(curpos
);
395 // remove spaces before/after
396 curpos
= Buffer
.find_first_not_of(" \t\r");
398 Buffer
.erase(0, curpos
);
399 curpos
= Buffer
.find_last_not_of(" \t\r");
400 if (curpos
!= std::string::npos
)
401 Buffer
.erase(curpos
+ 1);
407 std::string
const LineType
= Buffer
.substr(0, Buffer
.find_first_of(" \t\v"));
408 if (LineType
.empty() || LineType
== Buffer
)
409 return _error
->Error(_("Malformed line %u in source list %s (type)"),CurLine
,File
.c_str());
411 Type
*Parse
= Type::GetType(LineType
.c_str());
413 return _error
->Error(_("Type '%s' is not known on line %u in source list %s"),LineType
.c_str(),CurLine
,File
.c_str());
415 if (Parse
->ParseLine(SrcList
, Buffer
.c_str() + LineType
.length(), CurLine
, File
) == false)
421 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/
422 // ---------------------------------------------------------------------
423 /* Returns: the number of stanzas parsed*/
424 bool pkgSourceList::ParseFileDeb822(string
const &File
)
426 // see if we can read the file
427 FileFd
Fd(File
, FileFd::ReadOnly
);
428 pkgTagFile
Sources(&Fd
, pkgTagFile::SUPPORT_COMMENTS
);
429 if (Fd
.IsOpen() == false || Fd
.Failed())
430 return _error
->Error(_("Malformed stanza %u in source list %s (type)"),0,File
.c_str());
435 while (Sources
.Step(Tags
) == true)
438 if(Tags
.Exists("Types") == false)
439 return _error
->Error(_("Malformed stanza %u in source list %s (type)"),i
,File
.c_str());
441 string
const types
= Tags
.FindS("Types");
442 std::vector
<std::string
> const list_types
= VectorizeString(types
, ' ');
443 for (std::vector
<std::string
>::const_iterator I
= list_types
.begin();
444 I
!= list_types
.end(); ++I
)
446 Type
*Parse
= Type::GetType((*I
).c_str());
449 _error
->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I
).c_str(),i
,Fd
.Name().c_str());
453 if (!Parse
->ParseStanza(SrcList
, Tags
, i
, Fd
))
460 // SourceList::FindIndex - Get the index associated with a file /*{{{*/
461 static bool FindInIndexFileContainer(std::vector
<pkgIndexFile
*> const &Cont
, pkgCache::PkgFileIterator
const &File
, pkgIndexFile
*&Found
)
463 auto const J
= std::find_if(Cont
.begin(), Cont
.end(), [&File
](pkgIndexFile
const * const J
) {
464 return J
->FindInCache(*File
.Cache()) == File
;
473 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File
,
474 pkgIndexFile
*&Found
) const
476 for (const_iterator I
= SrcList
.begin(); I
!= SrcList
.end(); ++I
)
477 if (FindInIndexFileContainer(*(*I
)->GetIndexFiles(), File
, Found
))
480 return FindInIndexFileContainer(VolatileFiles
, File
, Found
);
483 // SourceList::GetIndexes - Load the index files into the downloader /*{{{*/
484 // ---------------------------------------------------------------------
486 bool pkgSourceList::GetIndexes(pkgAcquire
*Owner
, bool GetAll
) const
488 for (const_iterator I
= SrcList
.begin(); I
!= SrcList
.end(); ++I
)
489 if ((*I
)->GetIndexes(Owner
,GetAll
) == false)
494 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
495 // SourceList::ReadSourceDir - Read a directory with sources files
496 // Based on ReadConfigDir() /*{{{*/
497 // ---------------------------------------------------------------------
499 bool pkgSourceList::ReadSourceDir(string
const &Dir
)
501 std::vector
<std::string
> ext
;
502 ext
.push_back("list");
503 ext
.push_back("sources");
504 std::vector
<std::string
> const List
= GetListOfFilesInDir(Dir
, ext
, true);
507 for (vector
<string
>::const_iterator I
= List
.begin(); I
!= List
.end(); ++I
)
508 if (ReadAppend(*I
) == false)
514 // GetLastModified() /*{{{*/
515 // ---------------------------------------------------------------------
517 time_t pkgSourceList::GetLastModifiedTime()
521 string Main
= _config
->FindFile("Dir::Etc::sourcelist");
522 string Parts
= _config
->FindDir("Dir::Etc::sourceparts");
525 if (DirectoryExists(Parts
) == true)
526 List
= GetListOfFilesInDir(Parts
, "list", true);
528 // calculate the time
529 std::vector
<time_t> modtimes
;
530 modtimes
.reserve(1 + List
.size());
531 modtimes
.push_back(GetModificationTime(Main
));
532 std::transform(List
.begin(), List
.end(), std::back_inserter(modtimes
), GetModificationTime
);
533 auto const maxmtime
= std::max_element(modtimes
.begin(), modtimes
.end());
537 std::vector
<pkgIndexFile
*> pkgSourceList::GetVolatileFiles() const /*{{{*/
539 return VolatileFiles
;
542 void pkgSourceList::AddVolatileFile(pkgIndexFile
* const File
) /*{{{*/
545 VolatileFiles
.push_back(File
);
548 static bool fileNameMatches(std::string
const &filename
, std::string
const &idxtype
)/*{{{*/
550 for (auto && type
: APT::Configuration::getCompressionTypes())
552 if (type
== "uncompressed")
554 if (filename
== idxtype
|| APT::String::Endswith(filename
, '_' + idxtype
))
557 else if (filename
== idxtype
+ '.' + type
||
558 APT::String::Endswith(filename
, '_' + idxtype
+ '.' + type
))
564 bool pkgSourceList::AddVolatileFile(std::string
const &File
, std::vector
<std::string
> * const VolatileCmdL
)/*{{{*/
566 // Note: FileExists matches directories and links, too!
567 if (File
.empty() || FileExists(File
) == false)
570 std::string
const ext
= flExtension(File
);
571 // udeb is not included as installing it is usually a mistake rather than intended
572 if (ext
== "deb" || ext
== "ddeb")
573 AddVolatileFile(new debDebPkgFileIndex(File
));
574 else if (ext
== "dsc")
575 AddVolatileFile(new debDscFileIndex(File
));
576 else if (FileExists(flCombine(File
, "debian/control")))
577 AddVolatileFile(new debDscFileIndex(flCombine(File
, "debian/control")));
578 else if (ext
== "changes")
580 debDscRecordParser
changes(File
, nullptr);
581 std::vector
<pkgSrcRecords::File2
> fileslst
;
582 if (changes
.Files2(fileslst
) == false || fileslst
.empty())
584 auto const basedir
= flNotFile(File
);
585 for (auto && file
: fileslst
)
587 auto const name
= flCombine(basedir
, file
.Path
);
588 AddVolatileFile(name
, VolatileCmdL
);
589 if (file
.Hashes
.VerifyFile(name
) == false)
590 return _error
->Error("The file %s does not match with the hashes in the %s file!", name
.c_str(), File
.c_str());
596 auto const filename
= flNotDir(File
);
597 auto const Target
= IndexTarget(File
, filename
, File
, "file:" + File
, false, true, {
598 { "FILENAME", File
},
599 { "REPO_URI", "file:" + flAbsPath(flNotFile(File
)) + '/' },
600 { "COMPONENT", "volatile-packages-file" },
602 if (fileNameMatches(filename
, "Packages"))
603 AddVolatileFile(new debPackagesIndex(Target
, true));
604 else if (fileNameMatches(filename
, "Sources"))
605 AddVolatileFile(new debSourcesIndex(Target
, true));
610 if (VolatileCmdL
!= nullptr)
611 VolatileCmdL
->push_back(File
);
614 bool pkgSourceList::AddVolatileFile(std::string
const &File
)
616 return AddVolatileFile(File
, nullptr);
619 void pkgSourceList::AddVolatileFiles(CommandLine
&CmdL
, std::vector
<std::string
> * const VolatileCmdL
)/*{{{*/
621 std::remove_if(CmdL
.FileList
+ 1, CmdL
.FileList
+ 1 + CmdL
.FileSize(), [&](char const * const I
) {
622 if (I
!= nullptr && (I
[0] == '/' || (I
[0] == '.' && ((I
[1] == '.' && I
[2] == '/') || I
[1] == '/'))))
624 if (AddVolatileFile(I
, VolatileCmdL
))
627 _error
->Error(_("Unsupported file %s given on commandline"), I
);
633 void pkgSourceList::AddVolatileFiles(CommandLine
&CmdL
, std::vector
<const char*> * const VolatileCmdL
)
635 std::remove_if(CmdL
.FileList
+ 1, CmdL
.FileList
+ 1 + CmdL
.FileSize(), [&](char const * const I
) {
636 if (I
!= nullptr && (I
[0] == '/' || (I
[0] == '.' && ((I
[1] == '.' && I
[2] == '/') || I
[1] == '/'))))
638 if (AddVolatileFile(I
))
640 if (VolatileCmdL
!= nullptr)
641 VolatileCmdL
->push_back(I
);
644 _error
->Error(_("Unsupported file %s given on commandline"), I
);