1 // -*- mode: cpp; mode: fold -*-
3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
4 /* ######################################################################
8 ##################################################################### */
10 // Include Files /*{{{*/
13 #include <apt-pkg/sourcelist.h>
14 #include <apt-pkg/cmndline.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
18 #include <apt-pkg/configuration.h>
19 #include <apt-pkg/metaindex.h>
20 #include <apt-pkg/indexfile.h>
21 #include <apt-pkg/tagfile.h>
22 #include <apt-pkg/pkgcache.h>
23 #include <apt-pkg/cacheiterators.h>
24 #include <apt-pkg/debindexfile.h>
41 // Global list of Items supported
42 static pkgSourceList::Type
*ItmList
[10];
43 pkgSourceList::Type
**pkgSourceList::Type::GlobalList
= ItmList
;
44 unsigned long pkgSourceList::Type::GlobalListLen
= 0;
46 // Type::Type - Constructor /*{{{*/
47 // ---------------------------------------------------------------------
48 /* Link this to the global list of items*/
49 pkgSourceList::Type::Type(char const * const pName
, char const * const pLabel
) : Name(pName
), Label(pLabel
)
51 ItmList
[GlobalListLen
] = this;
54 pkgSourceList::Type::~Type() {}
56 // Type::GetType - Get a specific meta for a given type /*{{{*/
57 // ---------------------------------------------------------------------
59 pkgSourceList::Type
*pkgSourceList::Type::GetType(const char *Type
)
61 for (unsigned I
= 0; I
!= GlobalListLen
; ++I
)
62 if (strcmp(GlobalList
[I
]->Name
,Type
) == 0)
67 // Type::FixupURI - Normalize the URI and check it.. /*{{{*/
68 // ---------------------------------------------------------------------
70 bool pkgSourceList::Type::FixupURI(string
&URI
) const
72 if (URI
.empty() == true)
75 if (URI
.find(':') == string::npos
)
78 URI
= SubstVar(URI
,"$(ARCH)",_config
->Find("APT::Architecture"));
80 // Make sure that the URI is / postfixed
81 if (URI
[URI
.size() - 1] != '/')
87 bool pkgSourceList::Type::ParseStanza(vector
<metaIndex
*> &List
, /*{{{*/
92 map
<string
, string
> Options
;
94 string Enabled
= Tags
.FindS("Enabled");
95 if (Enabled
.empty() == false && StringToBool(Enabled
) == false)
98 std::map
<char const * const, std::pair
<char const * const, bool> > mapping
;
99 #define APT_PLUSMINUS(X, Y) \
100 mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \
101 mapping.insert(std::make_pair(X "-Add", std::make_pair(Y "+", true))); \
102 mapping.insert(std::make_pair(X "-Remove", std::make_pair(Y "-", true)))
103 APT_PLUSMINUS("Architectures", "arch");
104 APT_PLUSMINUS("Languages", "lang");
105 APT_PLUSMINUS("Targets", "target");
107 mapping
.insert(std::make_pair("Trusted", std::make_pair("trusted", false)));
108 mapping
.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false)));
109 mapping
.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false)));
110 mapping
.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
111 mapping
.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
112 mapping
.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
113 mapping
.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));
115 for (std::map
<char const * const, std::pair
<char const * const, bool> >::const_iterator m
= mapping
.begin(); m
!= mapping
.end(); ++m
)
116 if (Tags
.Exists(m
->first
))
118 std::string option
= Tags
.FindS(m
->first
);
119 // for deb822 the " " is the delimiter, but the backend expects ","
120 if (m
->second
.second
== true)
121 std::replace(option
.begin(), option
.end(), ' ', ',');
122 Options
[m
->second
.first
] = option
;
127 strprintf(entry
, "%s:%i", Fd
.Name().c_str(), i
);
128 Options
["sourceslist-entry"] = entry
;
131 // now create one item per suite/section
132 string Suite
= Tags
.FindS("Suites");
133 Suite
= SubstVar(Suite
,"$(ARCH)",_config
->Find("APT::Architecture"));
134 string
const Component
= Tags
.FindS("Components");
135 string
const URIS
= Tags
.FindS("URIs");
137 std::vector
<std::string
> const list_uris
= VectorizeString(URIS
, ' ');
138 std::vector
<std::string
> const list_suite
= VectorizeString(Suite
, ' ');
139 std::vector
<std::string
> const list_comp
= VectorizeString(Component
, ' ');
141 if (list_uris
.empty())
142 // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
143 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "URI");
145 for (std::vector
<std::string
>::const_iterator U
= list_uris
.begin();
146 U
!= list_uris
.end(); ++U
)
148 std::string URI
= *U
;
149 if (U
->empty() || FixupURI(URI
) == false)
150 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "URI parse");
152 if (list_suite
.empty())
153 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "Suite");
155 for (std::vector
<std::string
>::const_iterator S
= list_suite
.begin();
156 S
!= list_suite
.end(); ++S
)
158 if (S
->empty() == false && (*S
)[S
->size() - 1] == '/')
160 if (list_comp
.empty() == false)
161 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "absolute Suite Component");
162 if (CreateItem(List
, URI
, *S
, "", Options
) == false)
167 if (list_comp
.empty())
168 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), i
, "sources", Fd
.Name().c_str(), "Component");
170 for (std::vector
<std::string
>::const_iterator C
= list_comp
.begin();
171 C
!= list_comp
.end(); ++C
)
173 if (CreateItem(List
, URI
, *S
, *C
, Options
) == false)
184 // Type::ParseLine - Parse a single line /*{{{*/
185 // ---------------------------------------------------------------------
186 /* This is a generic one that is the 'usual' format for sources.list
187 Weird types may override this. */
188 bool pkgSourceList::Type::ParseLine(vector
<metaIndex
*> &List
,
190 unsigned int const CurLine
,
191 string
const &File
) const
193 for (;Buffer
!= 0 && isspace(*Buffer
); ++Buffer
); // Skip whitespaces
195 // Parse option field if it exists
196 // e.g.: [ option1=value1 option2=value2 ]
197 map
<string
, string
> Options
;
200 strprintf(entry
, "%s:%i", File
.c_str(), CurLine
);
201 Options
["sourceslist-entry"] = entry
;
203 if (Buffer
!= 0 && Buffer
[0] == '[')
205 ++Buffer
; // ignore the [
206 for (;Buffer
!= 0 && isspace(*Buffer
); ++Buffer
); // Skip whitespaces
207 while (*Buffer
!= ']')
209 // get one option, e.g. option1=value1
211 if (ParseQuoteWord(Buffer
,option
) == false)
212 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] unparseable");
214 if (option
.length() < 3)
215 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] too short");
217 // accept options even if the last has no space before the ]-end marker
218 if (option
.at(option
.length()-1) == ']')
220 for (; *Buffer
!= ']'; --Buffer
);
221 option
.resize(option
.length()-1);
224 size_t const needle
= option
.find('=');
225 if (needle
== string::npos
)
226 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] not assignment");
228 string
const key
= string(option
, 0, needle
);
229 string
const value
= string(option
, needle
+ 1, option
.length());
231 if (key
.empty() == true)
232 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] no key");
234 if (value
.empty() == true)
235 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "[option] no value");
237 Options
[key
] = value
;
239 ++Buffer
; // ignore the ]
240 for (;Buffer
!= 0 && isspace(*Buffer
); ++Buffer
); // Skip whitespaces
247 if (ParseQuoteWord(Buffer
,URI
) == false)
248 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "URI");
249 if (ParseQuoteWord(Buffer
,Dist
) == false)
250 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "Suite");
252 if (FixupURI(URI
) == false)
253 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "URI parse");
255 // Check for an absolute dists specification.
256 if (Dist
.empty() == false && Dist
[Dist
.size() - 1] == '/')
258 if (ParseQuoteWord(Buffer
,Section
) == true)
259 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "absolute Suite Component");
260 Dist
= SubstVar(Dist
,"$(ARCH)",_config
->Find("APT::Architecture"));
261 return CreateItem(List
, URI
, Dist
, Section
, Options
);
264 // Grab the rest of the dists
265 if (ParseQuoteWord(Buffer
,Section
) == false)
266 return _error
->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine
, "list", File
.c_str(), "Component");
270 if (CreateItem(List
, URI
, Dist
, Section
, Options
) == false)
273 while (ParseQuoteWord(Buffer
,Section
) == true);
278 // SourceList::pkgSourceList - Constructors /*{{{*/
279 // ---------------------------------------------------------------------
281 pkgSourceList::pkgSourceList() : d(NULL
)
285 // SourceList::~pkgSourceList - Destructor /*{{{*/
286 // ---------------------------------------------------------------------
288 pkgSourceList::~pkgSourceList()
290 for (const_iterator I
= SrcList
.begin(); I
!= SrcList
.end(); ++I
)
293 for (auto F
= VolatileFiles
.begin(); F
!= VolatileFiles
.end(); ++F
)
295 VolatileFiles
.clear();
298 // SourceList::ReadMainList - Read the main source list from etc /*{{{*/
299 // ---------------------------------------------------------------------
301 bool pkgSourceList::ReadMainList()
303 // CNC:2003-03-03 - Multiple sources list support.
312 // CNC:2003-11-28 - Entries in sources.list have priority over
313 // entries in sources.list.d.
314 string Main
= _config
->FindFile("Dir::Etc::sourcelist");
315 string Parts
= _config
->FindDir("Dir::Etc::sourceparts");
317 if (RealFileExists(Main
) == true)
318 Res
&= ReadAppend(Main
);
319 else if (DirectoryExists(Parts
) == false)
320 // Only warn if there are no sources.list.d.
321 _error
->WarningE("DirectoryExists", _("Unable to read %s"), Parts
.c_str());
323 if (DirectoryExists(Parts
) == true)
324 Res
&= ReadSourceDir(Parts
);
325 else if (RealFileExists(Main
) == false)
326 // Only warn if there is no sources.list file.
327 _error
->WarningE("RealFileExists", _("Unable to read %s"), Main
.c_str());
332 // SourceList::Reset - Clear the sourcelist contents /*{{{*/
333 // ---------------------------------------------------------------------
335 void pkgSourceList::Reset()
337 for (const_iterator I
= SrcList
.begin(); I
!= SrcList
.end(); ++I
)
342 // SourceList::Read - Parse the sourcelist file /*{{{*/
343 // ---------------------------------------------------------------------
345 bool pkgSourceList::Read(string
const &File
)
348 return ReadAppend(File
);
351 // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/
352 // ---------------------------------------------------------------------
354 bool pkgSourceList::ReadAppend(string
const &File
)
356 if (flExtension(File
) == "sources")
357 return ParseFileDeb822(File
);
359 return ParseFileOldStyle(File
);
362 // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/
363 // ---------------------------------------------------------------------
365 bool pkgSourceList::ParseFileOldStyle(std::string
const &File
)
367 // Open the stream for reading
368 ifstream
F(File
.c_str(),ios::in
/*| ios::nocreate*/);
369 if (F
.fail() == true)
370 return _error
->Errno("ifstream::ifstream",_("Opening %s"),File
.c_str());
373 for (unsigned int CurLine
= 1; std::getline(F
, Buffer
); ++CurLine
)
377 while ((curpos
= Buffer
.find('#', curpos
)) != std::string::npos
)
379 size_t const openbrackets
= std::count(Buffer
.begin(), Buffer
.begin() + curpos
, '[');
380 size_t const closedbrackets
= std::count(Buffer
.begin(), Buffer
.begin() + curpos
, ']');
381 if (openbrackets
> closedbrackets
)
383 // a # in an option, unlikely, but oh well, it was supported so stick to it
387 Buffer
.erase(curpos
);
390 // remove spaces before/after
391 curpos
= Buffer
.find_first_not_of(" \t\r");
393 Buffer
.erase(0, curpos
);
394 curpos
= Buffer
.find_last_not_of(" \t\r");
395 if (curpos
!= std::string::npos
)
396 Buffer
.erase(curpos
+ 1);
402 std::string
const LineType
= Buffer
.substr(0, Buffer
.find_first_of(" \t\v"));
403 if (LineType
.empty() || LineType
== Buffer
)
404 return _error
->Error(_("Malformed line %u in source list %s (type)"),CurLine
,File
.c_str());
406 Type
*Parse
= Type::GetType(LineType
.c_str());
408 return _error
->Error(_("Type '%s' is not known on line %u in source list %s"),LineType
.c_str(),CurLine
,File
.c_str());
410 if (Parse
->ParseLine(SrcList
, Buffer
.c_str() + LineType
.length(), CurLine
, File
) == false)
416 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/
417 // ---------------------------------------------------------------------
418 /* Returns: the number of stanzas parsed*/
419 bool pkgSourceList::ParseFileDeb822(string
const &File
)
423 // see if we can read the file
424 FileFd
Fd(File
, FileFd::ReadOnly
);
425 pkgTagFile
Sources(&Fd
, pkgTagFile::SUPPORT_COMMENTS
);
426 if (Fd
.IsOpen() == false || Fd
.Failed())
427 return _error
->Error(_("Malformed stanza %u in source list %s (type)"),i
,File
.c_str());
431 while (Sources
.Step(Tags
) == true)
433 if(Tags
.Exists("Types") == false)
434 return _error
->Error(_("Malformed stanza %u in source list %s (type)"),i
,File
.c_str());
436 string
const types
= Tags
.FindS("Types");
437 std::vector
<std::string
> const list_types
= VectorizeString(types
, ' ');
438 for (std::vector
<std::string
>::const_iterator I
= list_types
.begin();
439 I
!= list_types
.end(); ++I
)
441 Type
*Parse
= Type::GetType((*I
).c_str());
444 _error
->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I
).c_str(),i
,Fd
.Name().c_str());
448 if (!Parse
->ParseStanza(SrcList
, Tags
, i
, Fd
))
457 // SourceList::FindIndex - Get the index associated with a file /*{{{*/
458 static bool FindInIndexFileContainer(std::vector
<pkgIndexFile
*> const &Cont
, pkgCache::PkgFileIterator
const &File
, pkgIndexFile
*&Found
)
460 auto const J
= std::find_if(Cont
.begin(), Cont
.end(), [&File
](pkgIndexFile
const * const J
) {
461 return J
->FindInCache(*File
.Cache()) == File
;
470 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File
,
471 pkgIndexFile
*&Found
) const
473 for (const_iterator I
= SrcList
.begin(); I
!= SrcList
.end(); ++I
)
474 if (FindInIndexFileContainer(*(*I
)->GetIndexFiles(), File
, Found
))
477 return FindInIndexFileContainer(VolatileFiles
, File
, Found
);
480 // SourceList::GetIndexes - Load the index files into the downloader /*{{{*/
481 // ---------------------------------------------------------------------
483 bool pkgSourceList::GetIndexes(pkgAcquire
*Owner
, bool GetAll
) const
485 for (const_iterator I
= SrcList
.begin(); I
!= SrcList
.end(); ++I
)
486 if ((*I
)->GetIndexes(Owner
,GetAll
) == false)
491 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
492 // SourceList::ReadSourceDir - Read a directory with sources files
493 // Based on ReadConfigDir() /*{{{*/
494 // ---------------------------------------------------------------------
496 bool pkgSourceList::ReadSourceDir(string
const &Dir
)
498 std::vector
<std::string
> ext
;
499 ext
.push_back("list");
500 ext
.push_back("sources");
501 std::vector
<std::string
> const List
= GetListOfFilesInDir(Dir
, ext
, true);
504 for (vector
<string
>::const_iterator I
= List
.begin(); I
!= List
.end(); ++I
)
505 if (ReadAppend(*I
) == false)
511 // GetLastModified() /*{{{*/
512 // ---------------------------------------------------------------------
514 time_t pkgSourceList::GetLastModifiedTime()
518 string Main
= _config
->FindFile("Dir::Etc::sourcelist");
519 string Parts
= _config
->FindDir("Dir::Etc::sourceparts");
522 if (DirectoryExists(Parts
) == true)
523 List
= GetListOfFilesInDir(Parts
, "list", true);
525 // calculate the time
526 std::vector
<time_t> modtimes
;
527 modtimes
.reserve(1 + List
.size());
528 modtimes
.push_back(GetModificationTime(Main
));
529 std::transform(List
.begin(), List
.end(), std::back_inserter(modtimes
), GetModificationTime
);
530 auto const maxmtime
= std::max_element(modtimes
.begin(), modtimes
.end());
534 std::vector
<pkgIndexFile
*> pkgSourceList::GetVolatileFiles() const /*{{{*/
536 return VolatileFiles
;
539 void pkgSourceList::AddVolatileFile(pkgIndexFile
* const File
) /*{{{*/
542 VolatileFiles
.push_back(File
);
545 bool pkgSourceList::AddVolatileFile(std::string
const &File
) /*{{{*/
547 // Note: FileExists matches directories and links, too!
548 if (File
.empty() || FileExists(File
) == false)
551 std::string
const ext
= flExtension(File
);
553 AddVolatileFile(new debDebPkgFileIndex(File
));
554 else if (ext
== "dsc")
555 AddVolatileFile(new debDscFileIndex(File
));
556 else if (FileExists(flCombine(File
, "debian/control")))
557 AddVolatileFile(new debDscFileIndex(flCombine(File
, "debian/control")));
564 void pkgSourceList::AddVolatileFiles(CommandLine
&CmdL
, std::vector
<const char*> * const VolatileCmdL
)/*{{{*/
566 std::remove_if(CmdL
.FileList
+ 1, CmdL
.FileList
+ 1 + CmdL
.FileSize(), [&](char const * const I
) {
567 if (I
!= nullptr && (I
[0] == '/' || (I
[0] == '.' && ((I
[1] == '.' && I
[2] == '/') || I
[1] == '/'))))
569 if (AddVolatileFile(I
))
571 if (VolatileCmdL
!= nullptr)
572 VolatileCmdL
->push_back(I
);
575 _error
->Error(_("Unsupported file %s given on commandline"), I
);