apt-pkg/sourcelist.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
   4 /* ######################################################################
   5
   6    List of Sources
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include<config.h>
  12
  13 #include <apt-pkg/sourcelist.h>
  14 #include <apt-pkg/error.h>
  15 #include <apt-pkg/fileutl.h>
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/metaindex.h>
  19 #include <apt-pkg/indexfile.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/pkgcache.h>
  22 #include <apt-pkg/cacheiterators.h>
  23
  24 #include <ctype.h>
  25 #include <stddef.h>
  26 #include <time.h>
  27 #include <cstring>
  28 #include <map>
  29 #include <string>
  30 #include <vector>
  31 #include <fstream>
  32 #include <algorithm>
  33
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Global list of Items supported
  40 static pkgSourceList::Type *ItmList[10];
  41 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
  42 unsigned long pkgSourceList::Type::GlobalListLen = 0;
  43
  44 // Type::Type - Constructor                                             /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* Link this to the global list of items*/
  47 pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
  48 {
  49    ItmList[GlobalListLen] = this;
  50    ++GlobalListLen;
  51 }
  52 pkgSourceList::Type::~Type() {}
  53                                                                         /*}}}*/
  54 // Type::GetType - Get a specific meta for a given type                 /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
  58 {
  59    for (unsigned I = 0; I != GlobalListLen; ++I)
  60       if (strcmp(GlobalList[I]->Name,Type) == 0)
  61          return GlobalList[I];
  62    return 0;
  63 }
  64                                                                         /*}}}*/
  65 // Type::FixupURI - Normalize the URI and check it..                    /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* */
  68 bool pkgSourceList::Type::FixupURI(string &URI) const
  69 {
  70    if (URI.empty() == true)
  71       return false;
  72
  73    if (URI.find(':') == string::npos)
  74       return false;
  75
  76    URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
  77
  78    // Make sure that the URI is / postfixed
  79    if (URI[URI.size() - 1] != '/')
  80       URI += '/';
  81
  82    return true;
  83 }
  84                                                                         /*}}}*/
  85 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,
  86                                       pkgTagSection &Tags,
  87                                       int i,
  88                                       FileFd &Fd)
  89 {
  90    map<string, string> Options;
  91
  92    string Enabled = Tags.FindS("Enabled");
  93    if (Enabled.size() > 0 && StringToBool(Enabled) == false)
  94       return true;
  95
  96    std::map<char const * const, char const * const> mapping;
  97 #define APT_PLUSMINUS(X, Y) \
  98    mapping.insert(std::make_pair(X, Y)); \
  99    mapping.insert(std::make_pair(X "Add", Y "+")); \
 100    mapping.insert(std::make_pair(X "Remove", Y "-"))
 101    APT_PLUSMINUS("Architectures", "arch");
 102    APT_PLUSMINUS("Languages", "lang");
 103    APT_PLUSMINUS("Targets", "target");
 104 #undef APT_PLUSMINUS
 105    mapping.insert(std::make_pair("Trusted", "trusted"));
 106    for (std::map<char const * const, char const * const>::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
 107       if (Tags.Exists(m->first))
 108       {
 109          // for deb822 the " " is the delimiter, but the backend expects ","
 110          std::string option = Tags.FindS(m->first);
 111          std::replace(option.begin(), option.end(), ' ', ',');
 112          Options[m->second] = option;
 113       }
 114
 115    // now create one item per suite/section
 116    string Suite = Tags.FindS("Suites");
 117    Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
 118    string const Section = Tags.FindS("Sections");
 119    string URIS = Tags.FindS("URIs");
 120
 121    std::vector<std::string> list_uris = StringSplit(URIS, " ");
 122    std::vector<std::string> list_dist = StringSplit(Suite, " ");
 123    std::vector<std::string> list_section = StringSplit(Section, " ");
 124
 125    for (std::vector<std::string>::const_iterator U = list_uris.begin();
 126         U != list_uris.end(); ++U)
 127    {
 128       std::string URI = (*U);
 129       if (!FixupURI(URI))
 130       {
 131          _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str());
 132          return false;
 133       }
 134
 135       for (std::vector<std::string>::const_iterator I = list_dist.begin();
 136            I != list_dist.end(); ++I)
 137       {
 138          for (std::vector<std::string>::const_iterator J = list_section.begin();
 139               J != list_section.end(); ++J)
 140          {
 141             if (CreateItem(List, URI, (*I), (*J), Options) == false)
 142             {
 143                return false;
 144             }
 145          }
 146       }
 147    }
 148    return true;
 149 }
 150
 151 // Type::ParseLine - Parse a single line                                /*{{{*/
 152 // ---------------------------------------------------------------------
 153 /* This is a generic one that is the 'usual' format for sources.list
 154    Weird types may override this. */
 155 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 156                                     const char *Buffer,
 157                                     unsigned long const &CurLine,
 158                                     string const &File) const
 159 {
 160    for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 161
 162    // Parse option field if it exists
 163    // e.g.: [ option1=value1 option2=value2 ]
 164    map<string, string> Options;
 165    if (Buffer != 0 && Buffer[0] == '[')
 166    {
 167       ++Buffer; // ignore the [
 168       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 169       while (*Buffer != ']')
 170       {
 171          // get one option, e.g. option1=value1
 172          string option;
 173          if (ParseQuoteWord(Buffer,option) == false)
 174             return _error->Error(_("Malformed line %lu in source list %s ([option] unparseable)"),CurLine,File.c_str());
 175
 176          if (option.length() < 3)
 177             return _error->Error(_("Malformed line %lu in source list %s ([option] too short)"),CurLine,File.c_str());
 178
 179          // accept options even if the last has no space before the ]-end marker
 180          if (option.at(option.length()-1) == ']')
 181          {
 182             for (; *Buffer != ']'; --Buffer);
 183             option.resize(option.length()-1);
 184          }
 185
 186          size_t const needle = option.find('=');
 187          if (needle == string::npos)
 188             return _error->Error(_("Malformed line %lu in source list %s ([%s] is not an assignment)"),CurLine,File.c_str(), option.c_str());
 189
 190          string const key = string(option, 0, needle);
 191          string const value = string(option, needle + 1, option.length());
 192
 193          if (key.empty() == true)
 194             return _error->Error(_("Malformed line %lu in source list %s ([%s] has no key)"),CurLine,File.c_str(), option.c_str());
 195
 196          if (value.empty() == true)
 197             return _error->Error(_("Malformed line %lu in source list %s ([%s] key %s has no value)"),CurLine,File.c_str(),option.c_str(),key.c_str());
 198
 199          Options[key] = value;
 200       }
 201       ++Buffer; // ignore the ]
 202       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 203    }
 204
 205    string URI;
 206    string Dist;
 207    string Section;
 208
 209    if (ParseQuoteWord(Buffer,URI) == false)
 210       return _error->Error(_("Malformed line %lu in source list %s (URI)"),CurLine,File.c_str());
 211    if (ParseQuoteWord(Buffer,Dist) == false)
 212       return _error->Error(_("Malformed line %lu in source list %s (dist)"),CurLine,File.c_str());
 213
 214    if (FixupURI(URI) == false)
 215       return _error->Error(_("Malformed line %lu in source list %s (URI parse)"),CurLine,File.c_str());
 216
 217    // Check for an absolute dists specification.
 218    if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
 219    {
 220       if (ParseQuoteWord(Buffer,Section) == true)
 221          return _error->Error(_("Malformed line %lu in source list %s (absolute dist)"),CurLine,File.c_str());
 222       Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
 223       return CreateItem(List, URI, Dist, Section, Options);
 224    }
 225
 226    // Grab the rest of the dists
 227    if (ParseQuoteWord(Buffer,Section) == false)
 228       return _error->Error(_("Malformed line %lu in source list %s (dist parse)"),CurLine,File.c_str());
 229
 230    do
 231    {
 232       if (CreateItem(List, URI, Dist, Section, Options) == false)
 233          return false;
 234    }
 235    while (ParseQuoteWord(Buffer,Section) == true);
 236
 237    return true;
 238 }
 239                                                                         /*}}}*/
 240 // SourceList::pkgSourceList - Constructors                             /*{{{*/
 241 // ---------------------------------------------------------------------
 242 /* */
 243 pkgSourceList::pkgSourceList() : d(NULL)
 244 {
 245 }
 246
 247 pkgSourceList::pkgSourceList(string File) : d(NULL)
 248 {
 249    Read(File);
 250 }
 251                                                                         /*}}}*/
 252 // SourceList::~pkgSourceList - Destructor                              /*{{{*/
 253 // ---------------------------------------------------------------------
 254 /* */
 255 pkgSourceList::~pkgSourceList()
 256 {
 257    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 258       delete *I;
 259 }
 260                                                                         /*}}}*/
 261 // SourceList::ReadMainList - Read the main source list from etc        /*{{{*/
 262 // ---------------------------------------------------------------------
 263 /* */
 264 bool pkgSourceList::ReadMainList()
 265 {
 266    // CNC:2003-03-03 - Multiple sources list support.
 267    bool Res = true;
 268 #if 0
 269    Res = ReadVendors();
 270    if (Res == false)
 271       return false;
 272 #endif
 273
 274    Reset();
 275    // CNC:2003-11-28 - Entries in sources.list have priority over
 276    //                  entries in sources.list.d.
 277    string Main = _config->FindFile("Dir::Etc::sourcelist");
 278    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 279
 280    if (RealFileExists(Main) == true)
 281       Res &= ReadAppend(Main);
 282    else if (DirectoryExists(Parts) == false)
 283       // Only warn if there are no sources.list.d.
 284       _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
 285
 286    if (DirectoryExists(Parts) == true)
 287       Res &= ReadSourceDir(Parts);
 288    else if (RealFileExists(Main) == false)
 289       // Only warn if there is no sources.list file.
 290       _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
 291
 292    return Res;
 293 }
 294                                                                         /*}}}*/
 295 // SourceList::Reset - Clear the sourcelist contents                    /*{{{*/
 296 // ---------------------------------------------------------------------
 297 /* */
 298 void pkgSourceList::Reset()
 299 {
 300    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 301       delete *I;
 302    SrcList.erase(SrcList.begin(),SrcList.end());
 303 }
 304                                                                         /*}}}*/
 305 // SourceList::Read - Parse the sourcelist file                         /*{{{*/
 306 // ---------------------------------------------------------------------
 307 /* */
 308 bool pkgSourceList::Read(string File)
 309 {
 310    Reset();
 311    return ReadAppend(File);
 312 }
 313                                                                         /*}}}*/
 314 // SourceList::ReadAppend - Parse a sourcelist file                     /*{{{*/
 315 // ---------------------------------------------------------------------
 316 /* */
 317 bool pkgSourceList::ReadAppend(string File)
 318 {
 319    if (_config->FindB("APT::Sources::Use-Deb822", false) == true)
 320    {
 321       int lines_parsed =ParseFileDeb822(File);
 322       if (lines_parsed < 0)
 323          return false;
 324       else if (lines_parsed > 0)
 325          return true;
 326       // no lines parsed  ... fall through and use old style parser
 327    }
 328    return ParseFileOldStyle(File);
 329 }
 330
 331 // SourceList::ReadFileOldStyle - Read Traditional style sources.list   /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* */
 334 bool pkgSourceList::ParseFileOldStyle(string File)
 335 {
 336    // Open the stream for reading
 337    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
 338    if (F.fail() == true)
 339       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
 340
 341    // CNC:2003-12-10 - 300 is too short.
 342    char Buffer[1024];
 343
 344    int CurLine = 0;
 345    while (F.eof() == false)
 346    {
 347       F.getline(Buffer,sizeof(Buffer));
 348       CurLine++;
 349       _strtabexpand(Buffer,sizeof(Buffer));
 350       if (F.fail() && !F.eof())
 351          return _error->Error(_("Line %u too long in source list %s."),
 352                               CurLine,File.c_str());
 353
 354
 355       char *I;
 356       // CNC:2003-02-20 - Do not break if '#' is inside [].
 357       for (I = Buffer; *I != 0 && *I != '#'; I++)
 358          if (*I == '[')
 359          {
 360             char *b_end = strchr(I + 1, ']');
 361             if (b_end != NULL)
 362                I = b_end;
 363          }
 364       *I = 0;
 365
 366       const char *C = _strstrip(Buffer);
 367
 368       // Comment or blank
 369       if (C[0] == '#' || C[0] == 0)
 370          continue;
 371
 372       // Grok it
 373       string LineType;
 374       if (ParseQuoteWord(C,LineType) == false)
 375          return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
 376
 377       Type *Parse = Type::GetType(LineType.c_str());
 378       if (Parse == 0)
 379          return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
 380
 381       if (Parse->ParseLine(SrcList, C, CurLine, File) == false)
 382          return false;
 383    }
 384    return true;
 385 }
 386                                                                         /*}}}*/
 387 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list        /*{{{*/
 388 // ---------------------------------------------------------------------
 389 /* Returns: the number of stanzas parsed*/
 390 int pkgSourceList::ParseFileDeb822(string File)
 391 {
 392    pkgTagSection Tags;
 393    unsigned int i=0;
 394
 395    // see if we can read the file
 396    _error->PushToStack();
 397    FileFd Fd(File, FileFd::ReadOnly);
 398    pkgTagFile Sources(&Fd);
 399    if (_error->PendingError() == true)
 400    {
 401       _error->RevertToStack();
 402       return 0;
 403    }
 404    _error->MergeWithStack();
 405
 406    // read step by step
 407    while (Sources.Step(Tags) == true)
 408    {
 409       if(!Tags.Exists("Types"))
 410          continue;
 411
 412       string const types = Tags.FindS("Types");
 413       std::vector<std::string> list_types = StringSplit(types, " ");
 414       for (std::vector<std::string>::const_iterator I = list_types.begin();
 415         I != list_types.end(); ++I)
 416       {
 417          Type *Parse = Type::GetType((*I).c_str());
 418          if (Parse == 0)
 419          {
 420             _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
 421             return -1;
 422          }
 423
 424          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
 425             return -1;
 426
 427          i++;
 428       }
 429    }
 430
 431    // we are done, return the number of stanzas read
 432    return i;
 433 }
 434                                                                         /*}}}*/
 435 // SourceList::FindIndex - Get the index associated with a file         /*{{{*/
 436 // ---------------------------------------------------------------------
 437 /* */
 438 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
 439                               pkgIndexFile *&Found) const
 440 {
 441    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 442    {
 443       vector<pkgIndexFile *> *Indexes = (*I)->GetIndexFiles();
 444       for (vector<pkgIndexFile *>::const_iterator J = Indexes->begin();
 445            J != Indexes->end(); ++J)
 446       {
 447          if ((*J)->FindInCache(*File.Cache()) == File)
 448          {
 449             Found = (*J);
 450             return true;
 451          }
 452       }
 453    }
 454
 455    return false;
 456 }
 457                                                                         /*}}}*/
 458 // SourceList::GetIndexes - Load the index files into the downloader    /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* */
 461 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 462 {
 463    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 464       if ((*I)->GetIndexes(Owner,GetAll) == false)
 465          return false;
 466    return true;
 467 }
 468                                                                         /*}}}*/
 469 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
 470 // SourceList::ReadSourceDir - Read a directory with sources files
 471 // Based on ReadConfigDir()                                             /*{{{*/
 472 // ---------------------------------------------------------------------
 473 /* */
 474 bool pkgSourceList::ReadSourceDir(string Dir)
 475 {
 476    vector<string> const List = GetListOfFilesInDir(Dir, "list", true);
 477
 478    // Read the files
 479    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 480       if (ReadAppend(*I) == false)
 481          return false;
 482    return true;
 483
 484 }
 485                                                                         /*}}}*/
 486 // GetLastModified()                                            /*{{{*/
 487 // ---------------------------------------------------------------------
 488 /* */
 489 time_t pkgSourceList::GetLastModifiedTime()
 490 {
 491    vector<string> List;
 492
 493    string Main = _config->FindFile("Dir::Etc::sourcelist");
 494    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 495
 496    // go over the parts
 497    if (DirectoryExists(Parts) == true)
 498       List = GetListOfFilesInDir(Parts, "list", true);
 499
 500    // calculate the time
 501    time_t mtime_sources = GetModificationTime(Main);
 502    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 503       mtime_sources = std::max(mtime_sources, GetModificationTime(*I));
 504
 505    return mtime_sources;
 506 }
 507                                                                         /*}}}*/
 508