apt-pkg/sourcelist.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
   4 /* ######################################################################
   5
   6    List of Sources
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include<config.h>
  12
  13 #include <apt-pkg/sourcelist.h>
  14 #include <apt-pkg/error.h>
  15 #include <apt-pkg/fileutl.h>
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/metaindex.h>
  19 #include <apt-pkg/indexfile.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/pkgcache.h>
  22 #include <apt-pkg/cacheiterators.h>
  23
  24 #include <ctype.h>
  25 #include <stddef.h>
  26 #include <time.h>
  27 #include <cstring>
  28 #include <map>
  29 #include <string>
  30 #include <vector>
  31 #include <fstream>
  32 #include <algorithm>
  33
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Global list of Items supported
  40 static pkgSourceList::Type *ItmList[10];
  41 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
  42 unsigned long pkgSourceList::Type::GlobalListLen = 0;
  43
  44 // Type::Type - Constructor                                             /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* Link this to the global list of items*/
  47 pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
  48 {
  49    ItmList[GlobalListLen] = this;
  50    ++GlobalListLen;
  51 }
  52 pkgSourceList::Type::~Type() {}
  53                                                                         /*}}}*/
  54 // Type::GetType - Get a specific meta for a given type                 /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
  58 {
  59    for (unsigned I = 0; I != GlobalListLen; ++I)
  60       if (strcmp(GlobalList[I]->Name,Type) == 0)
  61          return GlobalList[I];
  62    return 0;
  63 }
  64                                                                         /*}}}*/
  65 // Type::FixupURI - Normalize the URI and check it..                    /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* */
  68 bool pkgSourceList::Type::FixupURI(string &URI) const
  69 {
  70    if (URI.empty() == true)
  71       return false;
  72
  73    if (URI.find(':') == string::npos)
  74       return false;
  75
  76    URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
  77
  78    // Make sure that the URI is / postfixed
  79    if (URI[URI.size() - 1] != '/')
  80       URI += '/';
  81
  82    return true;
  83 }
  84                                                                         /*}}}*/
  85 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,        /*{{{*/
  86                                       pkgTagSection &Tags,
  87                                       unsigned int const i,
  88                                       FileFd &Fd)
  89 {
  90    map<string, string> Options;
  91
  92    string Enabled = Tags.FindS("Enabled");
  93    if (Enabled.empty() == false && StringToBool(Enabled) == false)
  94       return true;
  95
  96    std::map<char const * const, char const * const> mapping;
  97 #define APT_PLUSMINUS(X, Y) \
  98    mapping.insert(std::make_pair(X, Y)); \
  99    mapping.insert(std::make_pair(X "Add", Y "+")); \
 100    mapping.insert(std::make_pair(X "Remove", Y "-"))
 101    APT_PLUSMINUS("Architectures", "arch");
 102    APT_PLUSMINUS("Languages", "lang");
 103    APT_PLUSMINUS("Targets", "target");
 104 #undef APT_PLUSMINUS
 105    mapping.insert(std::make_pair("Trusted", "trusted"));
 106    mapping.insert(std::make_pair("Check-Valid-Until", "check-valid-until"));
 107    mapping.insert(std::make_pair("Valid-Until-Min", "valid-until-min"));
 108    mapping.insert(std::make_pair("Valid-Until-Max", "valid-until-max"));
 109
 110    for (std::map<char const * const, char const * const>::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
 111       if (Tags.Exists(m->first))
 112       {
 113          // for deb822 the " " is the delimiter, but the backend expects ","
 114          std::string option = Tags.FindS(m->first);
 115          std::replace(option.begin(), option.end(), ' ', ',');
 116          Options[m->second] = option;
 117       }
 118
 119    // now create one item per suite/section
 120    string Suite = Tags.FindS("Suites");
 121    Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
 122    string const Component = Tags.FindS("Components");
 123    string const URIS = Tags.FindS("URIs");
 124
 125    std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
 126    std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
 127    std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
 128
 129    if (list_uris.empty())
 130       // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
 131       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
 132
 133    for (std::vector<std::string>::const_iterator U = list_uris.begin();
 134         U != list_uris.end(); ++U)
 135    {
 136       std::string URI = *U;
 137       if (U->empty() || FixupURI(URI) == false)
 138          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
 139
 140       if (list_suite.empty())
 141          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
 142
 143       for (std::vector<std::string>::const_iterator S = list_suite.begin();
 144            S != list_suite.end(); ++S)
 145       {
 146          if (S->empty() == false && (*S)[S->size() - 1] == '/')
 147          {
 148             if (list_comp.empty() == false)
 149                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
 150             if (CreateItem(List, URI, *S, "", Options) == false)
 151                return false;
 152          }
 153          else
 154          {
 155             if (list_comp.empty())
 156                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
 157
 158             for (std::vector<std::string>::const_iterator C = list_comp.begin();
 159                   C != list_comp.end(); ++C)
 160             {
 161                if (CreateItem(List, URI, *S, *C, Options) == false)
 162                {
 163                   return false;
 164                }
 165             }
 166          }
 167       }
 168    }
 169    return true;
 170 }
 171                                                                         /*}}}*/
 172 // Type::ParseLine - Parse a single line                                /*{{{*/
 173 // ---------------------------------------------------------------------
 174 /* This is a generic one that is the 'usual' format for sources.list
 175    Weird types may override this. */
 176 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 177                                     const char *Buffer,
 178                                     unsigned int const CurLine,
 179                                     string const &File) const
 180 {
 181    for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 182
 183    // Parse option field if it exists
 184    // e.g.: [ option1=value1 option2=value2 ]
 185    map<string, string> Options;
 186    if (Buffer != 0 && Buffer[0] == '[')
 187    {
 188       ++Buffer; // ignore the [
 189       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 190       while (*Buffer != ']')
 191       {
 192          // get one option, e.g. option1=value1
 193          string option;
 194          if (ParseQuoteWord(Buffer,option) == false)
 195             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
 196
 197          if (option.length() < 3)
 198             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
 199
 200          // accept options even if the last has no space before the ]-end marker
 201          if (option.at(option.length()-1) == ']')
 202          {
 203             for (; *Buffer != ']'; --Buffer);
 204             option.resize(option.length()-1);
 205          }
 206
 207          size_t const needle = option.find('=');
 208          if (needle == string::npos)
 209             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
 210
 211          string const key = string(option, 0, needle);
 212          string const value = string(option, needle + 1, option.length());
 213
 214          if (key.empty() == true)
 215             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
 216
 217          if (value.empty() == true)
 218             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
 219
 220          Options[key] = value;
 221       }
 222       ++Buffer; // ignore the ]
 223       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 224    }
 225
 226    string URI;
 227    string Dist;
 228    string Section;
 229
 230    if (ParseQuoteWord(Buffer,URI) == false)
 231       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
 232    if (ParseQuoteWord(Buffer,Dist) == false)
 233       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
 234
 235    if (FixupURI(URI) == false)
 236       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
 237
 238    // Check for an absolute dists specification.
 239    if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
 240    {
 241       if (ParseQuoteWord(Buffer,Section) == true)
 242          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
 243       Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
 244       return CreateItem(List, URI, Dist, Section, Options);
 245    }
 246
 247    // Grab the rest of the dists
 248    if (ParseQuoteWord(Buffer,Section) == false)
 249       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
 250
 251    do
 252    {
 253       if (CreateItem(List, URI, Dist, Section, Options) == false)
 254          return false;
 255    }
 256    while (ParseQuoteWord(Buffer,Section) == true);
 257
 258    return true;
 259 }
 260                                                                         /*}}}*/
 261 // SourceList::pkgSourceList - Constructors                             /*{{{*/
 262 // ---------------------------------------------------------------------
 263 /* */
 264 pkgSourceList::pkgSourceList() : d(NULL)
 265 {
 266 }
 267                                                                         /*}}}*/
 268 // SourceList::~pkgSourceList - Destructor                              /*{{{*/
 269 // ---------------------------------------------------------------------
 270 /* */
 271 pkgSourceList::~pkgSourceList()
 272 {
 273    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 274       delete *I;
 275 }
 276                                                                         /*}}}*/
 277 // SourceList::ReadMainList - Read the main source list from etc        /*{{{*/
 278 // ---------------------------------------------------------------------
 279 /* */
 280 bool pkgSourceList::ReadMainList()
 281 {
 282    // CNC:2003-03-03 - Multiple sources list support.
 283    bool Res = true;
 284 #if 0
 285    Res = ReadVendors();
 286    if (Res == false)
 287       return false;
 288 #endif
 289
 290    Reset();
 291    // CNC:2003-11-28 - Entries in sources.list have priority over
 292    //                  entries in sources.list.d.
 293    string Main = _config->FindFile("Dir::Etc::sourcelist");
 294    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 295
 296    if (RealFileExists(Main) == true)
 297       Res &= ReadAppend(Main);
 298    else if (DirectoryExists(Parts) == false)
 299       // Only warn if there are no sources.list.d.
 300       _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
 301
 302    if (DirectoryExists(Parts) == true)
 303       Res &= ReadSourceDir(Parts);
 304    else if (RealFileExists(Main) == false)
 305       // Only warn if there is no sources.list file.
 306       _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
 307
 308    return Res;
 309 }
 310                                                                         /*}}}*/
 311 // SourceList::Reset - Clear the sourcelist contents                    /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* */
 314 void pkgSourceList::Reset()
 315 {
 316    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 317       delete *I;
 318    SrcList.erase(SrcList.begin(),SrcList.end());
 319 }
 320                                                                         /*}}}*/
 321 // SourceList::Read - Parse the sourcelist file                         /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* */
 324 bool pkgSourceList::Read(string const &File)
 325 {
 326    Reset();
 327    return ReadAppend(File);
 328 }
 329                                                                         /*}}}*/
 330 // SourceList::ReadAppend - Parse a sourcelist file                     /*{{{*/
 331 // ---------------------------------------------------------------------
 332 /* */
 333 bool pkgSourceList::ReadAppend(string const &File)
 334 {
 335    if (flExtension(File) == "sources")
 336       return ParseFileDeb822(File);
 337    else
 338       return ParseFileOldStyle(File);
 339 }
 340
 341 // SourceList::ReadFileOldStyle - Read Traditional style sources.list   /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* */
 344 bool pkgSourceList::ParseFileOldStyle(std::string const &File)
 345 {
 346    // Open the stream for reading
 347    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
 348    if (F.fail() == true)
 349       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
 350
 351    std::string Buffer;
 352    for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
 353    {
 354       // remove comments
 355       size_t curpos = 0;
 356       while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
 357       {
 358          size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
 359          size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
 360          if (openbrackets > closedbrackets)
 361          {
 362             // a # in an option, unlikely, but oh well, it was supported so stick to it
 363             ++curpos;
 364             continue;
 365          }
 366          Buffer.erase(curpos);
 367          break;
 368       }
 369       // remove spaces before/after
 370       curpos = Buffer.find_first_not_of(" \t\r");
 371       if (curpos != 0)
 372          Buffer.erase(0, curpos);
 373       curpos = Buffer.find_last_not_of(" \t\r");
 374       if (curpos != std::string::npos)
 375          Buffer.erase(curpos + 1);
 376
 377       if (Buffer.empty())
 378          continue;
 379
 380       // Grok it
 381       std::string const LineType = Buffer.substr(0, Buffer.find(' '));
 382       if (LineType.empty() || LineType == Buffer)
 383          return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
 384
 385       Type *Parse = Type::GetType(LineType.c_str());
 386       if (Parse == 0)
 387          return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
 388
 389       if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
 390          return false;
 391    }
 392    return true;
 393 }
 394                                                                         /*}}}*/
 395 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list        /*{{{*/
 396 // ---------------------------------------------------------------------
 397 /* Returns: the number of stanzas parsed*/
 398 bool pkgSourceList::ParseFileDeb822(string const &File)
 399 {
 400    pkgUserTagSection Tags;
 401    unsigned int i = 1;
 402
 403    // see if we can read the file
 404    FileFd Fd(File, FileFd::ReadOnly);
 405    pkgTagFile Sources(&Fd);
 406    if (_error->PendingError() == true)
 407       return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 408
 409    // read step by step
 410    while (Sources.Step(Tags) == true)
 411    {
 412       if(Tags.Exists("Types") == false)
 413          return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 414
 415       string const types = Tags.FindS("Types");
 416       std::vector<std::string> const list_types = VectorizeString(types, ' ');
 417       for (std::vector<std::string>::const_iterator I = list_types.begin();
 418         I != list_types.end(); ++I)
 419       {
 420          Type *Parse = Type::GetType((*I).c_str());
 421          if (Parse == 0)
 422          {
 423             _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
 424             return false;
 425          }
 426
 427          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
 428             return false;
 429
 430          ++i;
 431       }
 432    }
 433    return true;
 434 }
 435                                                                         /*}}}*/
 436 // SourceList::FindIndex - Get the index associated with a file         /*{{{*/
 437 // ---------------------------------------------------------------------
 438 /* */
 439 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
 440                               pkgIndexFile *&Found) const
 441 {
 442    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 443    {
 444       vector<pkgIndexFile *> *Indexes = (*I)->GetIndexFiles();
 445       for (vector<pkgIndexFile *>::const_iterator J = Indexes->begin();
 446            J != Indexes->end(); ++J)
 447       {
 448          if ((*J)->FindInCache(*File.Cache()) == File)
 449          {
 450             Found = (*J);
 451             return true;
 452          }
 453       }
 454    }
 455
 456    return false;
 457 }
 458                                                                         /*}}}*/
 459 // SourceList::GetIndexes - Load the index files into the downloader    /*{{{*/
 460 // ---------------------------------------------------------------------
 461 /* */
 462 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 463 {
 464    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 465       if ((*I)->GetIndexes(Owner,GetAll) == false)
 466          return false;
 467    return true;
 468 }
 469                                                                         /*}}}*/
 470 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
 471 // SourceList::ReadSourceDir - Read a directory with sources files
 472 // Based on ReadConfigDir()                                             /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* */
 475 bool pkgSourceList::ReadSourceDir(string const &Dir)
 476 {
 477    std::vector<std::string> ext;
 478    ext.push_back("list");
 479    ext.push_back("sources");
 480    std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
 481
 482    // Read the files
 483    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 484       if (ReadAppend(*I) == false)
 485          return false;
 486    return true;
 487
 488 }
 489                                                                         /*}}}*/
 490 // GetLastModified()                                            /*{{{*/
 491 // ---------------------------------------------------------------------
 492 /* */
 493 time_t pkgSourceList::GetLastModifiedTime()
 494 {
 495    vector<string> List;
 496
 497    string Main = _config->FindFile("Dir::Etc::sourcelist");
 498    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 499
 500    // go over the parts
 501    if (DirectoryExists(Parts) == true)
 502       List = GetListOfFilesInDir(Parts, "list", true);
 503
 504    // calculate the time
 505    time_t mtime_sources = GetModificationTime(Main);
 506    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 507       mtime_sources = std::max(mtime_sources, GetModificationTime(*I));
 508
 509    return mtime_sources;
 510 }
 511                                                                         /*}}}*/
 512