apt-pkg/sourcelist.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
   4 /* ######################################################################
   5
   6    List of Sources
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include<config.h>
  12
  13 #include <apt-pkg/sourcelist.h>
  14 #include <apt-pkg/error.h>
  15 #include <apt-pkg/fileutl.h>
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/metaindex.h>
  19 #include <apt-pkg/indexfile.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/pkgcache.h>
  22 #include <apt-pkg/cacheiterators.h>
  23
  24 #include <ctype.h>
  25 #include <stddef.h>
  26 #include <time.h>
  27 #include <cstring>
  28 #include <map>
  29 #include <string>
  30 #include <vector>
  31 #include <fstream>
  32 #include <algorithm>
  33
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Global list of Items supported
  40 static pkgSourceList::Type *ItmList[10];
  41 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
  42 unsigned long pkgSourceList::Type::GlobalListLen = 0;
  43
  44 // Type::Type - Constructor                                             /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* Link this to the global list of items*/
  47 pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
  48 {
  49    ItmList[GlobalListLen] = this;
  50    ++GlobalListLen;
  51 }
  52 pkgSourceList::Type::~Type() {}
  53                                                                         /*}}}*/
  54 // Type::GetType - Get a specific meta for a given type                 /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
  58 {
  59    for (unsigned I = 0; I != GlobalListLen; ++I)
  60       if (strcmp(GlobalList[I]->Name,Type) == 0)
  61          return GlobalList[I];
  62    return 0;
  63 }
  64                                                                         /*}}}*/
  65 // Type::FixupURI - Normalize the URI and check it..                    /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* */
  68 bool pkgSourceList::Type::FixupURI(string &URI) const
  69 {
  70    if (URI.empty() == true)
  71       return false;
  72
  73    if (URI.find(':') == string::npos)
  74       return false;
  75
  76    URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
  77
  78    // Make sure that the URI is / postfixed
  79    if (URI[URI.size() - 1] != '/')
  80       URI += '/';
  81
  82    return true;
  83 }
  84                                                                         /*}}}*/
  85 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,        /*{{{*/
  86                                       pkgTagSection &Tags,
  87                                       unsigned int const i,
  88                                       FileFd &Fd)
  89 {
  90    map<string, string> Options;
  91
  92    string Enabled = Tags.FindS("Enabled");
  93    if (Enabled.empty() == false && StringToBool(Enabled) == false)
  94       return true;
  95
  96    std::map<char const * const, char const * const> mapping;
  97 #define APT_PLUSMINUS(X, Y) \
  98    mapping.insert(std::make_pair(X, Y)); \
  99    mapping.insert(std::make_pair(X "Add", Y "+")); \
 100    mapping.insert(std::make_pair(X "Remove", Y "-"))
 101    APT_PLUSMINUS("Architectures", "arch");
 102    APT_PLUSMINUS("Languages", "lang");
 103    APT_PLUSMINUS("Targets", "target");
 104 #undef APT_PLUSMINUS
 105    mapping.insert(std::make_pair("Trusted", "trusted"));
 106    for (std::map<char const * const, char const * const>::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
 107       if (Tags.Exists(m->first))
 108       {
 109          // for deb822 the " " is the delimiter, but the backend expects ","
 110          std::string option = Tags.FindS(m->first);
 111          std::replace(option.begin(), option.end(), ' ', ',');
 112          Options[m->second] = option;
 113       }
 114
 115    // now create one item per suite/section
 116    string Suite = Tags.FindS("Suites");
 117    Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
 118    string const Component = Tags.FindS("Components");
 119    string const URIS = Tags.FindS("URIs");
 120
 121    std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
 122    std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
 123    std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
 124
 125    if (list_uris.empty())
 126       // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
 127       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
 128
 129    for (std::vector<std::string>::const_iterator U = list_uris.begin();
 130         U != list_uris.end(); ++U)
 131    {
 132       std::string URI = *U;
 133       if (U->empty() || FixupURI(URI) == false)
 134          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
 135
 136       if (list_suite.empty())
 137          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
 138
 139       for (std::vector<std::string>::const_iterator S = list_suite.begin();
 140            S != list_suite.end(); ++S)
 141       {
 142          if (S->empty() == false && (*S)[S->size() - 1] == '/')
 143          {
 144             if (list_comp.empty() == false)
 145                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
 146             if (CreateItem(List, URI, *S, "", Options) == false)
 147                return false;
 148          }
 149          else
 150          {
 151             if (list_comp.empty())
 152                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
 153
 154             for (std::vector<std::string>::const_iterator C = list_comp.begin();
 155                   C != list_comp.end(); ++C)
 156             {
 157                if (CreateItem(List, URI, *S, *C, Options) == false)
 158                {
 159                   return false;
 160                }
 161             }
 162          }
 163       }
 164    }
 165    return true;
 166 }
 167                                                                         /*}}}*/
 168 // Type::ParseLine - Parse a single line                                /*{{{*/
 169 // ---------------------------------------------------------------------
 170 /* This is a generic one that is the 'usual' format for sources.list
 171    Weird types may override this. */
 172 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 173                                     const char *Buffer,
 174                                     unsigned int const CurLine,
 175                                     string const &File) const
 176 {
 177    for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 178
 179    // Parse option field if it exists
 180    // e.g.: [ option1=value1 option2=value2 ]
 181    map<string, string> Options;
 182    if (Buffer != 0 && Buffer[0] == '[')
 183    {
 184       ++Buffer; // ignore the [
 185       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 186       while (*Buffer != ']')
 187       {
 188          // get one option, e.g. option1=value1
 189          string option;
 190          if (ParseQuoteWord(Buffer,option) == false)
 191             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
 192
 193          if (option.length() < 3)
 194             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
 195
 196          // accept options even if the last has no space before the ]-end marker
 197          if (option.at(option.length()-1) == ']')
 198          {
 199             for (; *Buffer != ']'; --Buffer);
 200             option.resize(option.length()-1);
 201          }
 202
 203          size_t const needle = option.find('=');
 204          if (needle == string::npos)
 205             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
 206
 207          string const key = string(option, 0, needle);
 208          string const value = string(option, needle + 1, option.length());
 209
 210          if (key.empty() == true)
 211             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
 212
 213          if (value.empty() == true)
 214             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
 215
 216          Options[key] = value;
 217       }
 218       ++Buffer; // ignore the ]
 219       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 220    }
 221
 222    string URI;
 223    string Dist;
 224    string Section;
 225
 226    if (ParseQuoteWord(Buffer,URI) == false)
 227       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
 228    if (ParseQuoteWord(Buffer,Dist) == false)
 229       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
 230
 231    if (FixupURI(URI) == false)
 232       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
 233
 234    // Check for an absolute dists specification.
 235    if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
 236    {
 237       if (ParseQuoteWord(Buffer,Section) == true)
 238          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
 239       Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
 240       return CreateItem(List, URI, Dist, Section, Options);
 241    }
 242
 243    // Grab the rest of the dists
 244    if (ParseQuoteWord(Buffer,Section) == false)
 245       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
 246
 247    do
 248    {
 249       if (CreateItem(List, URI, Dist, Section, Options) == false)
 250          return false;
 251    }
 252    while (ParseQuoteWord(Buffer,Section) == true);
 253
 254    return true;
 255 }
 256                                                                         /*}}}*/
 257 // SourceList::pkgSourceList - Constructors                             /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* */
 260 pkgSourceList::pkgSourceList() : d(NULL)
 261 {
 262 }
 263                                                                         /*}}}*/
 264 // SourceList::~pkgSourceList - Destructor                              /*{{{*/
 265 // ---------------------------------------------------------------------
 266 /* */
 267 pkgSourceList::~pkgSourceList()
 268 {
 269    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 270       delete *I;
 271 }
 272                                                                         /*}}}*/
 273 // SourceList::ReadMainList - Read the main source list from etc        /*{{{*/
 274 // ---------------------------------------------------------------------
 275 /* */
 276 bool pkgSourceList::ReadMainList()
 277 {
 278    // CNC:2003-03-03 - Multiple sources list support.
 279    bool Res = true;
 280 #if 0
 281    Res = ReadVendors();
 282    if (Res == false)
 283       return false;
 284 #endif
 285
 286    Reset();
 287    // CNC:2003-11-28 - Entries in sources.list have priority over
 288    //                  entries in sources.list.d.
 289    string Main = _config->FindFile("Dir::Etc::sourcelist");
 290    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 291
 292    if (RealFileExists(Main) == true)
 293       Res &= ReadAppend(Main);
 294    else if (DirectoryExists(Parts) == false)
 295       // Only warn if there are no sources.list.d.
 296       _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
 297
 298    if (DirectoryExists(Parts) == true)
 299       Res &= ReadSourceDir(Parts);
 300    else if (RealFileExists(Main) == false)
 301       // Only warn if there is no sources.list file.
 302       _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
 303
 304    return Res;
 305 }
 306                                                                         /*}}}*/
 307 // SourceList::Reset - Clear the sourcelist contents                    /*{{{*/
 308 // ---------------------------------------------------------------------
 309 /* */
 310 void pkgSourceList::Reset()
 311 {
 312    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 313       delete *I;
 314    SrcList.erase(SrcList.begin(),SrcList.end());
 315 }
 316                                                                         /*}}}*/
 317 // SourceList::Read - Parse the sourcelist file                         /*{{{*/
 318 // ---------------------------------------------------------------------
 319 /* */
 320 bool pkgSourceList::Read(string const &File)
 321 {
 322    Reset();
 323    return ReadAppend(File);
 324 }
 325                                                                         /*}}}*/
 326 // SourceList::ReadAppend - Parse a sourcelist file                     /*{{{*/
 327 // ---------------------------------------------------------------------
 328 /* */
 329 bool pkgSourceList::ReadAppend(string const &File)
 330 {
 331    if (flExtension(File) == "sources")
 332       return ParseFileDeb822(File);
 333    else
 334       return ParseFileOldStyle(File);
 335 }
 336
 337 // SourceList::ReadFileOldStyle - Read Traditional style sources.list   /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* */
 340 bool pkgSourceList::ParseFileOldStyle(std::string const &File)
 341 {
 342    // Open the stream for reading
 343    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
 344    if (F.fail() == true)
 345       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
 346
 347    std::string Buffer;
 348    for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
 349    {
 350       // remove comments
 351       size_t curpos = 0;
 352       while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
 353       {
 354          size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
 355          size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
 356          if (openbrackets > closedbrackets)
 357          {
 358             // a # in an option, unlikely, but oh well, it was supported so stick to it
 359             ++curpos;
 360             continue;
 361          }
 362          Buffer.erase(curpos);
 363          break;
 364       }
 365       // remove spaces before/after
 366       curpos = Buffer.find_first_not_of(" \t\r");
 367       if (curpos != 0)
 368          Buffer.erase(0, curpos);
 369       curpos = Buffer.find_last_not_of(" \t\r");
 370       if (curpos != std::string::npos)
 371          Buffer.erase(curpos + 1);
 372
 373       if (Buffer.empty())
 374          continue;
 375
 376       // Grok it
 377       std::string const LineType = Buffer.substr(0, Buffer.find(' '));
 378       if (LineType.empty() || LineType == Buffer)
 379          return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
 380
 381       Type *Parse = Type::GetType(LineType.c_str());
 382       if (Parse == 0)
 383          return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
 384
 385       if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
 386          return false;
 387    }
 388    return true;
 389 }
 390                                                                         /*}}}*/
 391 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list        /*{{{*/
 392 // ---------------------------------------------------------------------
 393 /* Returns: the number of stanzas parsed*/
 394 bool pkgSourceList::ParseFileDeb822(string const &File)
 395 {
 396    pkgUserTagSection Tags;
 397    unsigned int i = 1;
 398
 399    // see if we can read the file
 400    FileFd Fd(File, FileFd::ReadOnly);
 401    pkgTagFile Sources(&Fd);
 402    if (_error->PendingError() == true)
 403       return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 404
 405    // read step by step
 406    while (Sources.Step(Tags) == true)
 407    {
 408       if(Tags.Exists("Types") == false)
 409          return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 410
 411       string const types = Tags.FindS("Types");
 412       std::vector<std::string> const list_types = VectorizeString(types, ' ');
 413       for (std::vector<std::string>::const_iterator I = list_types.begin();
 414         I != list_types.end(); ++I)
 415       {
 416          Type *Parse = Type::GetType((*I).c_str());
 417          if (Parse == 0)
 418          {
 419             _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
 420             return false;
 421          }
 422
 423          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
 424             return false;
 425
 426          ++i;
 427       }
 428    }
 429    return true;
 430 }
 431                                                                         /*}}}*/
 432 // SourceList::FindIndex - Get the index associated with a file         /*{{{*/
 433 // ---------------------------------------------------------------------
 434 /* */
 435 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
 436                               pkgIndexFile *&Found) const
 437 {
 438    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 439    {
 440       vector<pkgIndexFile *> *Indexes = (*I)->GetIndexFiles();
 441       for (vector<pkgIndexFile *>::const_iterator J = Indexes->begin();
 442            J != Indexes->end(); ++J)
 443       {
 444          if ((*J)->FindInCache(*File.Cache()) == File)
 445          {
 446             Found = (*J);
 447             return true;
 448          }
 449       }
 450    }
 451
 452    return false;
 453 }
 454                                                                         /*}}}*/
 455 // SourceList::GetIndexes - Load the index files into the downloader    /*{{{*/
 456 // ---------------------------------------------------------------------
 457 /* */
 458 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 459 {
 460    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 461       if ((*I)->GetIndexes(Owner,GetAll) == false)
 462          return false;
 463    return true;
 464 }
 465                                                                         /*}}}*/
 466 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
 467 // SourceList::ReadSourceDir - Read a directory with sources files
 468 // Based on ReadConfigDir()                                             /*{{{*/
 469 // ---------------------------------------------------------------------
 470 /* */
 471 bool pkgSourceList::ReadSourceDir(string const &Dir)
 472 {
 473    std::vector<std::string> ext;
 474    ext.push_back("list");
 475    ext.push_back("sources");
 476    std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
 477
 478    // Read the files
 479    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 480       if (ReadAppend(*I) == false)
 481          return false;
 482    return true;
 483
 484 }
 485                                                                         /*}}}*/
 486 // GetLastModified()                                            /*{{{*/
 487 // ---------------------------------------------------------------------
 488 /* */
 489 time_t pkgSourceList::GetLastModifiedTime()
 490 {
 491    vector<string> List;
 492
 493    string Main = _config->FindFile("Dir::Etc::sourcelist");
 494    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 495
 496    // go over the parts
 497    if (DirectoryExists(Parts) == true)
 498       List = GetListOfFilesInDir(Parts, "list", true);
 499
 500    // calculate the time
 501    time_t mtime_sources = GetModificationTime(Main);
 502    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 503       mtime_sources = std::max(mtime_sources, GetModificationTime(*I));
 504
 505    return mtime_sources;
 506 }
 507                                                                         /*}}}*/
 508