apt-pkg/sourcelist.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
   4 /* ######################################################################
   5
   6    List of Sources
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include<config.h>
  12
  13 #include <apt-pkg/sourcelist.h>
  14 #include <apt-pkg/error.h>
  15 #include <apt-pkg/fileutl.h>
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/metaindex.h>
  19 #include <apt-pkg/indexfile.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/pkgcache.h>
  22 #include <apt-pkg/cacheiterators.h>
  23
  24 #include <ctype.h>
  25 #include <stddef.h>
  26 #include <time.h>
  27 #include <cstring>
  28 #include <map>
  29 #include <string>
  30 #include <vector>
  31 #include <fstream>
  32 #include <algorithm>
  33
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Global list of Items supported
  40 static pkgSourceList::Type *ItmList[10];
  41 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
  42 unsigned long pkgSourceList::Type::GlobalListLen = 0;
  43
  44 // Type::Type - Constructor                                             /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* Link this to the global list of items*/
  47 pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
  48 {
  49    ItmList[GlobalListLen] = this;
  50    ++GlobalListLen;
  51 }
  52 pkgSourceList::Type::~Type() {}
  53                                                                         /*}}}*/
  54 // Type::GetType - Get a specific meta for a given type                 /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
  58 {
  59    for (unsigned I = 0; I != GlobalListLen; ++I)
  60       if (strcmp(GlobalList[I]->Name,Type) == 0)
  61          return GlobalList[I];
  62    return 0;
  63 }
  64                                                                         /*}}}*/
  65 // Type::FixupURI - Normalize the URI and check it..                    /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* */
  68 bool pkgSourceList::Type::FixupURI(string &URI) const
  69 {
  70    if (URI.empty() == true)
  71       return false;
  72
  73    if (URI.find(':') == string::npos)
  74       return false;
  75
  76    URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
  77
  78    // Make sure that the URI is / postfixed
  79    if (URI[URI.size() - 1] != '/')
  80       URI += '/';
  81
  82    return true;
  83 }
  84                                                                         /*}}}*/
  85 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,        /*{{{*/
  86                                       pkgTagSection &Tags,
  87                                       unsigned int const i,
  88                                       FileFd &Fd)
  89 {
  90    map<string, string> Options;
  91
  92    string Enabled = Tags.FindS("Enabled");
  93    if (Enabled.empty() == false && StringToBool(Enabled) == false)
  94       return true;
  95
  96    std::map<char const * const, std::pair<char const * const, bool> > mapping;
  97 #define APT_PLUSMINUS(X, Y) \
  98    mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \
  99    mapping.insert(std::make_pair(X "Add", std::make_pair(Y "+", true))); \
 100    mapping.insert(std::make_pair(X "Remove", std::make_pair(Y "-", true)))
 101    APT_PLUSMINUS("Architectures", "arch");
 102    APT_PLUSMINUS("Languages", "lang");
 103    APT_PLUSMINUS("Targets", "target");
 104 #undef APT_PLUSMINUS
 105    mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false)));
 106    mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false)));
 107    mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false)));
 108    mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
 109    mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
 110    mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
 111
 112    for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
 113       if (Tags.Exists(m->first))
 114       {
 115          std::string option = Tags.FindS(m->first);
 116          // for deb822 the " " is the delimiter, but the backend expects ","
 117          if (m->second.second == true)
 118             std::replace(option.begin(), option.end(), ' ', ',');
 119          Options[m->second.first] = option;
 120       }
 121
 122    {
 123       std::string entry;
 124       strprintf(entry, "%s:%i", Fd.Name().c_str(), i);
 125       Options["sourceslist-entry"] = entry;
 126    }
 127
 128    // now create one item per suite/section
 129    string Suite = Tags.FindS("Suites");
 130    Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
 131    string const Component = Tags.FindS("Components");
 132    string const URIS = Tags.FindS("URIs");
 133
 134    std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
 135    std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
 136    std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
 137
 138    if (list_uris.empty())
 139       // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
 140       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
 141
 142    for (std::vector<std::string>::const_iterator U = list_uris.begin();
 143         U != list_uris.end(); ++U)
 144    {
 145       std::string URI = *U;
 146       if (U->empty() || FixupURI(URI) == false)
 147          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
 148
 149       if (list_suite.empty())
 150          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
 151
 152       for (std::vector<std::string>::const_iterator S = list_suite.begin();
 153            S != list_suite.end(); ++S)
 154       {
 155          if (S->empty() == false && (*S)[S->size() - 1] == '/')
 156          {
 157             if (list_comp.empty() == false)
 158                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
 159             if (CreateItem(List, URI, *S, "", Options) == false)
 160                return false;
 161          }
 162          else
 163          {
 164             if (list_comp.empty())
 165                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
 166
 167             for (std::vector<std::string>::const_iterator C = list_comp.begin();
 168                   C != list_comp.end(); ++C)
 169             {
 170                if (CreateItem(List, URI, *S, *C, Options) == false)
 171                {
 172                   return false;
 173                }
 174             }
 175          }
 176       }
 177    }
 178    return true;
 179 }
 180                                                                         /*}}}*/
 181 // Type::ParseLine - Parse a single line                                /*{{{*/
 182 // ---------------------------------------------------------------------
 183 /* This is a generic one that is the 'usual' format for sources.list
 184    Weird types may override this. */
 185 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 186                                     const char *Buffer,
 187                                     unsigned int const CurLine,
 188                                     string const &File) const
 189 {
 190    for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 191
 192    // Parse option field if it exists
 193    // e.g.: [ option1=value1 option2=value2 ]
 194    map<string, string> Options;
 195    {
 196       std::string entry;
 197       strprintf(entry, "%s:%i", File.c_str(), CurLine);
 198       Options["sourceslist-entry"] = entry;
 199    }
 200    if (Buffer != 0 && Buffer[0] == '[')
 201    {
 202       ++Buffer; // ignore the [
 203       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 204       while (*Buffer != ']')
 205       {
 206          // get one option, e.g. option1=value1
 207          string option;
 208          if (ParseQuoteWord(Buffer,option) == false)
 209             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
 210
 211          if (option.length() < 3)
 212             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
 213
 214          // accept options even if the last has no space before the ]-end marker
 215          if (option.at(option.length()-1) == ']')
 216          {
 217             for (; *Buffer != ']'; --Buffer);
 218             option.resize(option.length()-1);
 219          }
 220
 221          size_t const needle = option.find('=');
 222          if (needle == string::npos)
 223             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
 224
 225          string const key = string(option, 0, needle);
 226          string const value = string(option, needle + 1, option.length());
 227
 228          if (key.empty() == true)
 229             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
 230
 231          if (value.empty() == true)
 232             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
 233
 234          Options[key] = value;
 235       }
 236       ++Buffer; // ignore the ]
 237       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 238    }
 239
 240    string URI;
 241    string Dist;
 242    string Section;
 243
 244    if (ParseQuoteWord(Buffer,URI) == false)
 245       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
 246    if (ParseQuoteWord(Buffer,Dist) == false)
 247       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
 248
 249    if (FixupURI(URI) == false)
 250       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
 251
 252    // Check for an absolute dists specification.
 253    if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
 254    {
 255       if (ParseQuoteWord(Buffer,Section) == true)
 256          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
 257       Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
 258       return CreateItem(List, URI, Dist, Section, Options);
 259    }
 260
 261    // Grab the rest of the dists
 262    if (ParseQuoteWord(Buffer,Section) == false)
 263       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
 264
 265    do
 266    {
 267       if (CreateItem(List, URI, Dist, Section, Options) == false)
 268          return false;
 269    }
 270    while (ParseQuoteWord(Buffer,Section) == true);
 271
 272    return true;
 273 }
 274                                                                         /*}}}*/
 275 // SourceList::pkgSourceList - Constructors                             /*{{{*/
 276 // ---------------------------------------------------------------------
 277 /* */
 278 pkgSourceList::pkgSourceList() : d(NULL)
 279 {
 280 }
 281                                                                         /*}}}*/
 282 // SourceList::~pkgSourceList - Destructor                              /*{{{*/
 283 // ---------------------------------------------------------------------
 284 /* */
 285 pkgSourceList::~pkgSourceList()
 286 {
 287    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 288       delete *I;
 289    SrcList.clear();
 290    for (auto  F = VolatileFiles.begin(); F != VolatileFiles.end(); ++F)
 291       delete (*F);
 292    VolatileFiles.clear();
 293 }
 294                                                                         /*}}}*/
 295 // SourceList::ReadMainList - Read the main source list from etc        /*{{{*/
 296 // ---------------------------------------------------------------------
 297 /* */
 298 bool pkgSourceList::ReadMainList()
 299 {
 300    // CNC:2003-03-03 - Multiple sources list support.
 301    bool Res = true;
 302 #if 0
 303    Res = ReadVendors();
 304    if (Res == false)
 305       return false;
 306 #endif
 307
 308    Reset();
 309    // CNC:2003-11-28 - Entries in sources.list have priority over
 310    //                  entries in sources.list.d.
 311    string Main = _config->FindFile("Dir::Etc::sourcelist");
 312    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 313
 314    if (RealFileExists(Main) == true)
 315       Res &= ReadAppend(Main);
 316    else if (DirectoryExists(Parts) == false)
 317       // Only warn if there are no sources.list.d.
 318       _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
 319
 320    if (DirectoryExists(Parts) == true)
 321       Res &= ReadSourceDir(Parts);
 322    else if (RealFileExists(Main) == false)
 323       // Only warn if there is no sources.list file.
 324       _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
 325
 326    return Res;
 327 }
 328                                                                         /*}}}*/
 329 // SourceList::Reset - Clear the sourcelist contents                    /*{{{*/
 330 // ---------------------------------------------------------------------
 331 /* */
 332 void pkgSourceList::Reset()
 333 {
 334    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 335       delete *I;
 336    SrcList.clear();
 337 }
 338                                                                         /*}}}*/
 339 // SourceList::Read - Parse the sourcelist file                         /*{{{*/
 340 // ---------------------------------------------------------------------
 341 /* */
 342 bool pkgSourceList::Read(string const &File)
 343 {
 344    Reset();
 345    return ReadAppend(File);
 346 }
 347                                                                         /*}}}*/
 348 // SourceList::ReadAppend - Parse a sourcelist file                     /*{{{*/
 349 // ---------------------------------------------------------------------
 350 /* */
 351 bool pkgSourceList::ReadAppend(string const &File)
 352 {
 353    if (flExtension(File) == "sources")
 354       return ParseFileDeb822(File);
 355    else
 356       return ParseFileOldStyle(File);
 357 }
 358                                                                         /*}}}*/
 359 // SourceList::ReadFileOldStyle - Read Traditional style sources.list   /*{{{*/
 360 // ---------------------------------------------------------------------
 361 /* */
 362 bool pkgSourceList::ParseFileOldStyle(std::string const &File)
 363 {
 364    // Open the stream for reading
 365    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
 366    if (F.fail() == true)
 367       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
 368
 369    std::string Buffer;
 370    for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
 371    {
 372       // remove comments
 373       size_t curpos = 0;
 374       while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
 375       {
 376          size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
 377          size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
 378          if (openbrackets > closedbrackets)
 379          {
 380             // a # in an option, unlikely, but oh well, it was supported so stick to it
 381             ++curpos;
 382             continue;
 383          }
 384          Buffer.erase(curpos);
 385          break;
 386       }
 387       // remove spaces before/after
 388       curpos = Buffer.find_first_not_of(" \t\r");
 389       if (curpos != 0)
 390          Buffer.erase(0, curpos);
 391       curpos = Buffer.find_last_not_of(" \t\r");
 392       if (curpos != std::string::npos)
 393          Buffer.erase(curpos + 1);
 394
 395       if (Buffer.empty())
 396          continue;
 397
 398       // Grok it
 399       std::string const LineType = Buffer.substr(0, Buffer.find_first_of(" \t\v"));
 400       if (LineType.empty() || LineType == Buffer)
 401          return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
 402
 403       Type *Parse = Type::GetType(LineType.c_str());
 404       if (Parse == 0)
 405          return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
 406
 407       if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
 408          return false;
 409    }
 410    return true;
 411 }
 412                                                                         /*}}}*/
 413 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list        /*{{{*/
 414 // ---------------------------------------------------------------------
 415 /* Returns: the number of stanzas parsed*/
 416 bool pkgSourceList::ParseFileDeb822(string const &File)
 417 {
 418    pkgUserTagSection Tags;
 419    unsigned int i = 1;
 420
 421    // see if we can read the file
 422    FileFd Fd(File, FileFd::ReadOnly);
 423    pkgTagFile Sources(&Fd);
 424    if (_error->PendingError() == true)
 425       return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 426
 427    // read step by step
 428    while (Sources.Step(Tags) == true)
 429    {
 430       if(Tags.Exists("Types") == false)
 431          return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 432
 433       string const types = Tags.FindS("Types");
 434       std::vector<std::string> const list_types = VectorizeString(types, ' ');
 435       for (std::vector<std::string>::const_iterator I = list_types.begin();
 436         I != list_types.end(); ++I)
 437       {
 438          Type *Parse = Type::GetType((*I).c_str());
 439          if (Parse == 0)
 440          {
 441             _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
 442             return false;
 443          }
 444
 445          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
 446             return false;
 447
 448          ++i;
 449       }
 450    }
 451    return true;
 452 }
 453                                                                         /*}}}*/
 454 // SourceList::FindIndex - Get the index associated with a file         /*{{{*/
 455 static bool FindInIndexFileContainer(std::vector<pkgIndexFile *> const &Cont, pkgCache::PkgFileIterator const &File, pkgIndexFile *&Found)
 456 {
 457    auto const J = std::find_if(Cont.begin(), Cont.end(), [&File](pkgIndexFile const * const J) {
 458          return J->FindInCache(*File.Cache()) == File;
 459    });
 460    if (J != Cont.end())
 461    {
 462       Found = (*J);
 463       return true;
 464    }
 465    return false;
 466 }
 467 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
 468                               pkgIndexFile *&Found) const
 469 {
 470    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 471       if (FindInIndexFileContainer(*(*I)->GetIndexFiles(), File, Found))
 472          return true;
 473
 474    return FindInIndexFileContainer(VolatileFiles, File, Found);
 475 }
 476                                                                         /*}}}*/
 477 // SourceList::GetIndexes - Load the index files into the downloader    /*{{{*/
 478 // ---------------------------------------------------------------------
 479 /* */
 480 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 481 {
 482    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 483       if ((*I)->GetIndexes(Owner,GetAll) == false)
 484          return false;
 485    return true;
 486 }
 487                                                                         /*}}}*/
 488 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
 489 // SourceList::ReadSourceDir - Read a directory with sources files
 490 // Based on ReadConfigDir()                                             /*{{{*/
 491 // ---------------------------------------------------------------------
 492 /* */
 493 bool pkgSourceList::ReadSourceDir(string const &Dir)
 494 {
 495    std::vector<std::string> ext;
 496    ext.push_back("list");
 497    ext.push_back("sources");
 498    std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
 499
 500    // Read the files
 501    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 502       if (ReadAppend(*I) == false)
 503          return false;
 504    return true;
 505
 506 }
 507                                                                         /*}}}*/
 508 // GetLastModified()                                            /*{{{*/
 509 // ---------------------------------------------------------------------
 510 /* */
 511 time_t pkgSourceList::GetLastModifiedTime()
 512 {
 513    vector<string> List;
 514
 515    string Main = _config->FindFile("Dir::Etc::sourcelist");
 516    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 517
 518    // go over the parts
 519    if (DirectoryExists(Parts) == true)
 520       List = GetListOfFilesInDir(Parts, "list", true);
 521
 522    // calculate the time
 523    std::vector<time_t> modtimes;
 524    modtimes.reserve(1 + List.size());
 525    modtimes.push_back(GetModificationTime(Main));
 526    std::transform(List.begin(), List.end(), std::back_inserter(modtimes), GetModificationTime);
 527    auto const maxmtime = std::max_element(modtimes.begin(), modtimes.end());
 528    return *maxmtime;
 529 }
 530                                                                         /*}}}*/
 531 std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const      /*{{{*/
 532 {
 533    return VolatileFiles;
 534 }
 535                                                                         /*}}}*/
 536 void pkgSourceList::AddVolatileFile(pkgIndexFile * const File)          /*{{{*/
 537 {
 538    if (File != NULL)
 539       VolatileFiles.push_back(File);
 540 }
 541                                                                         /*}}}*/