apt-pkg/sourcelist.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
   4 /* ######################################################################
   5
   6    List of Sources
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include<config.h>
  12
  13 #include <apt-pkg/sourcelist.h>
  14 #include <apt-pkg/error.h>
  15 #include <apt-pkg/fileutl.h>
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/metaindex.h>
  19 #include <apt-pkg/indexfile.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/pkgcache.h>
  22 #include <apt-pkg/cacheiterators.h>
  23
  24 #include <ctype.h>
  25 #include <stddef.h>
  26 #include <time.h>
  27 #include <cstring>
  28 #include <map>
  29 #include <string>
  30 #include <vector>
  31 #include <fstream>
  32 #include <algorithm>
  33
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Global list of Items supported
  40 static  pkgSourceList::Type *ItmList[10];
  41 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
  42 unsigned long pkgSourceList::Type::GlobalListLen = 0;
  43
  44 // Type::Type - Constructor                                             /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* Link this to the global list of items*/
  47 pkgSourceList::Type::Type() : Name(NULL), Label(NULL)
  48 {
  49    ItmList[GlobalListLen] = this;
  50    GlobalListLen++;
  51 }
  52                                                                         /*}}}*/
  53 // Type::GetType - Get a specific meta for a given type                 /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
  57 {
  58    for (unsigned I = 0; I != GlobalListLen; I++)
  59       if (strcmp(GlobalList[I]->Name,Type) == 0)
  60          return GlobalList[I];
  61    return 0;
  62 }
  63                                                                         /*}}}*/
  64 // Type::FixupURI - Normalize the URI and check it..                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 bool pkgSourceList::Type::FixupURI(string &URI) const
  68 {
  69    if (URI.empty() == true)
  70       return false;
  71
  72    if (URI.find(':') == string::npos)
  73       return false;
  74
  75    URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
  76
  77    // Make sure that the URI is / postfixed
  78    if (URI[URI.size() - 1] != '/')
  79       URI += '/';
  80
  81    return true;
  82 }
  83                                                                         /*}}}*/
  84 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,
  85                                       pkgTagSection &Tags,
  86                                       int i,
  87                                       FileFd &Fd)
  88 {
  89    map<string, string> Options;
  90
  91    string Enabled = Tags.FindS("Enabled");
  92    if (Enabled.size() > 0 && StringToBool(Enabled) == false)
  93       return true;
  94
  95    // Define external/internal options
  96    const char* option_deb822[] = {
  97       "Architectures", "Architectures-Add", "Architectures-Remove", "Trusted",
  98    };
  99    const char* option_internal[] = {
 100       "arch", "arch+", "arch-", "trusted",
 101    };
 102    for (unsigned int j=0; j < sizeof(option_deb822)/sizeof(char*); j++)
 103       if (Tags.Exists(option_deb822[j]))
 104       {
 105          // for deb822 the " " is the delimiter, but the backend expects ","
 106          std::string option = Tags.FindS(option_deb822[j]);
 107          std::replace(option.begin(), option.end(), ' ', ',');
 108          Options[option_internal[j]] = option;
 109       }
 110
 111    // now create one item per suite/section
 112    string Suite = Tags.FindS("Suites");
 113    Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
 114    string const Section = Tags.FindS("Sections");
 115    string URIS = Tags.FindS("URIs");
 116
 117    std::vector<std::string> list_uris = StringSplit(URIS, " ");
 118    std::vector<std::string> list_dist = StringSplit(Suite, " ");
 119    std::vector<std::string> list_section = StringSplit(Section, " ");
 120
 121    for (std::vector<std::string>::const_iterator U = list_uris.begin();
 122         U != list_uris.end(); ++U)
 123    {
 124       std::string URI = (*U);
 125       if (!FixupURI(URI))
 126       {
 127          _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str());
 128          return false;
 129       }
 130
 131       for (std::vector<std::string>::const_iterator I = list_dist.begin();
 132            I != list_dist.end(); ++I)
 133       {
 134          for (std::vector<std::string>::const_iterator J = list_section.begin();
 135               J != list_section.end(); ++J)
 136          {
 137             if (CreateItem(List, URI, (*I), (*J), Options) == false)
 138             {
 139                return false;
 140             }
 141          }
 142       }
 143    }
 144    return true;
 145 }
 146
 147 // Type::ParseLine - Parse a single line                                /*{{{*/
 148 // ---------------------------------------------------------------------
 149 /* This is a generic one that is the 'usual' format for sources.list
 150    Weird types may override this. */
 151 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 152                                     const char *Buffer,
 153                                     unsigned long const &CurLine,
 154                                     string const &File) const
 155 {
 156    for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 157
 158    // Parse option field if it exists
 159    // e.g.: [ option1=value1 option2=value2 ]
 160    map<string, string> Options;
 161    if (Buffer != 0 && Buffer[0] == '[')
 162    {
 163       ++Buffer; // ignore the [
 164       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 165       while (*Buffer != ']')
 166       {
 167          // get one option, e.g. option1=value1
 168          string option;
 169          if (ParseQuoteWord(Buffer,option) == false)
 170             return _error->Error(_("Malformed line %lu in source list %s ([option] unparseable)"),CurLine,File.c_str());
 171
 172          if (option.length() < 3)
 173             return _error->Error(_("Malformed line %lu in source list %s ([option] too short)"),CurLine,File.c_str());
 174
 175          // accept options even if the last has no space before the ]-end marker
 176          if (option.at(option.length()-1) == ']')
 177          {
 178             for (; *Buffer != ']'; --Buffer);
 179             option.resize(option.length()-1);
 180          }
 181
 182          size_t const needle = option.find('=');
 183          if (needle == string::npos)
 184             return _error->Error(_("Malformed line %lu in source list %s ([%s] is not an assignment)"),CurLine,File.c_str(), option.c_str());
 185
 186          string const key = string(option, 0, needle);
 187          string const value = string(option, needle + 1, option.length());
 188
 189          if (key.empty() == true)
 190             return _error->Error(_("Malformed line %lu in source list %s ([%s] has no key)"),CurLine,File.c_str(), option.c_str());
 191
 192          if (value.empty() == true)
 193             return _error->Error(_("Malformed line %lu in source list %s ([%s] key %s has no value)"),CurLine,File.c_str(),option.c_str(),key.c_str());
 194
 195          Options[key] = value;
 196       }
 197       ++Buffer; // ignore the ]
 198       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 199    }
 200
 201    string URI;
 202    string Dist;
 203    string Section;
 204
 205    if (ParseQuoteWord(Buffer,URI) == false)
 206       return _error->Error(_("Malformed line %lu in source list %s (URI)"),CurLine,File.c_str());
 207    if (ParseQuoteWord(Buffer,Dist) == false)
 208       return _error->Error(_("Malformed line %lu in source list %s (dist)"),CurLine,File.c_str());
 209
 210    if (FixupURI(URI) == false)
 211       return _error->Error(_("Malformed line %lu in source list %s (URI parse)"),CurLine,File.c_str());
 212
 213    // Check for an absolute dists specification.
 214    if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
 215    {
 216       if (ParseQuoteWord(Buffer,Section) == true)
 217          return _error->Error(_("Malformed line %lu in source list %s (absolute dist)"),CurLine,File.c_str());
 218       Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
 219       return CreateItem(List, URI, Dist, Section, Options);
 220    }
 221
 222    // Grab the rest of the dists
 223    if (ParseQuoteWord(Buffer,Section) == false)
 224       return _error->Error(_("Malformed line %lu in source list %s (dist parse)"),CurLine,File.c_str());
 225
 226    do
 227    {
 228       if (CreateItem(List, URI, Dist, Section, Options) == false)
 229          return false;
 230    }
 231    while (ParseQuoteWord(Buffer,Section) == true);
 232
 233    return true;
 234 }
 235                                                                         /*}}}*/
 236 // SourceList::pkgSourceList - Constructors                             /*{{{*/
 237 // ---------------------------------------------------------------------
 238 /* */
 239 pkgSourceList::pkgSourceList()
 240 {
 241 }
 242
 243 pkgSourceList::pkgSourceList(string File)
 244 {
 245    Read(File);
 246 }
 247                                                                         /*}}}*/
 248 // SourceList::~pkgSourceList - Destructor                              /*{{{*/
 249 // ---------------------------------------------------------------------
 250 /* */
 251 pkgSourceList::~pkgSourceList()
 252 {
 253    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 254       delete *I;
 255 }
 256                                                                         /*}}}*/
 257 // SourceList::ReadMainList - Read the main source list from etc        /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* */
 260 bool pkgSourceList::ReadMainList()
 261 {
 262    // CNC:2003-03-03 - Multiple sources list support.
 263    bool Res = true;
 264 #if 0
 265    Res = ReadVendors();
 266    if (Res == false)
 267       return false;
 268 #endif
 269
 270    Reset();
 271    // CNC:2003-11-28 - Entries in sources.list have priority over
 272    //                  entries in sources.list.d.
 273    string Main = _config->FindFile("Dir::Etc::sourcelist");
 274    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 275
 276    if (RealFileExists(Main) == true)
 277       Res &= ReadAppend(Main);
 278    else if (DirectoryExists(Parts) == false)
 279       // Only warn if there are no sources.list.d.
 280       _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
 281
 282    if (DirectoryExists(Parts) == true)
 283       Res &= ReadSourceDir(Parts);
 284    else if (RealFileExists(Main) == false)
 285       // Only warn if there is no sources.list file.
 286       _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
 287
 288    return Res;
 289 }
 290                                                                         /*}}}*/
 291 // SourceList::Reset - Clear the sourcelist contents                    /*{{{*/
 292 // ---------------------------------------------------------------------
 293 /* */
 294 void pkgSourceList::Reset()
 295 {
 296    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 297       delete *I;
 298    SrcList.erase(SrcList.begin(),SrcList.end());
 299 }
 300                                                                         /*}}}*/
 301 // SourceList::Read - Parse the sourcelist file                         /*{{{*/
 302 // ---------------------------------------------------------------------
 303 /* */
 304 bool pkgSourceList::Read(string File)
 305 {
 306    Reset();
 307    return ReadAppend(File);
 308 }
 309                                                                         /*}}}*/
 310 // SourceList::ReadAppend - Parse a sourcelist file                     /*{{{*/
 311 // ---------------------------------------------------------------------
 312 /* */
 313 bool pkgSourceList::ReadAppend(string File)
 314 {
 315    if (_config->FindB("APT::Sources::Use-Deb822", false) == true)
 316    {
 317       int lines_parsed =ParseFileDeb822(File);
 318       if (lines_parsed < 0)
 319          return false;
 320       else if (lines_parsed > 0)
 321          return true;
 322       // no lines parsed  ... fall through and use old style parser
 323    }
 324    return ParseFileOldStyle(File);
 325 }
 326
 327 // SourceList::ReadFileOldStyle - Read Traditional style sources.list   /*{{{*/
 328 // ---------------------------------------------------------------------
 329 /* */
 330 bool pkgSourceList::ParseFileOldStyle(string File)
 331 {
 332    // Open the stream for reading
 333    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
 334    if (F.fail() == true)
 335       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
 336
 337    // CNC:2003-12-10 - 300 is too short.
 338    char Buffer[1024];
 339
 340    int CurLine = 0;
 341    while (F.eof() == false)
 342    {
 343       F.getline(Buffer,sizeof(Buffer));
 344       CurLine++;
 345       _strtabexpand(Buffer,sizeof(Buffer));
 346       if (F.fail() && !F.eof())
 347          return _error->Error(_("Line %u too long in source list %s."),
 348                               CurLine,File.c_str());
 349
 350
 351       char *I;
 352       // CNC:2003-02-20 - Do not break if '#' is inside [].
 353       for (I = Buffer; *I != 0 && *I != '#'; I++)
 354          if (*I == '[')
 355          {
 356             char *b_end = strchr(I + 1, ']');
 357             if (b_end != NULL)
 358                I = b_end;
 359          }
 360       *I = 0;
 361
 362       const char *C = _strstrip(Buffer);
 363
 364       // Comment or blank
 365       if (C[0] == '#' || C[0] == 0)
 366          continue;
 367
 368       // Grok it
 369       string LineType;
 370       if (ParseQuoteWord(C,LineType) == false)
 371          return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
 372
 373       Type *Parse = Type::GetType(LineType.c_str());
 374       if (Parse == 0)
 375          return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
 376
 377       if (Parse->ParseLine(SrcList, C, CurLine, File) == false)
 378          return false;
 379    }
 380    return true;
 381 }
 382                                                                         /*}}}*/
 383 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list        /*{{{*/
 384 // ---------------------------------------------------------------------
 385 /* Returns: the number of stanzas parsed*/
 386 int pkgSourceList::ParseFileDeb822(string File)
 387 {
 388    pkgTagSection Tags;
 389    unsigned int i=0;
 390
 391    // see if we can read the file
 392    _error->PushToStack();
 393    FileFd Fd(File, FileFd::ReadOnly);
 394    pkgTagFile Sources(&Fd);
 395    if (_error->PendingError() == true)
 396    {
 397       _error->RevertToStack();
 398       return 0;
 399    }
 400    _error->MergeWithStack();
 401
 402    // read step by step
 403    while (Sources.Step(Tags) == true)
 404    {
 405       if(!Tags.Exists("Types"))
 406          continue;
 407
 408       string const types = Tags.FindS("Types");
 409       std::vector<std::string> list_types = StringSplit(types, " ");
 410       for (std::vector<std::string>::const_iterator I = list_types.begin();
 411         I != list_types.end(); ++I)
 412       {
 413          Type *Parse = Type::GetType((*I).c_str());
 414          if (Parse == 0)
 415          {
 416             _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
 417             return -1;
 418          }
 419
 420          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
 421             return -1;
 422
 423          i++;
 424       }
 425    }
 426
 427    // we are done, return the number of stanzas read
 428    return i;
 429 }
 430                                                                         /*}}}*/
 431 // SourceList::FindIndex - Get the index associated with a file         /*{{{*/
 432 // ---------------------------------------------------------------------
 433 /* */
 434 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
 435                               pkgIndexFile *&Found) const
 436 {
 437    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 438    {
 439       vector<pkgIndexFile *> *Indexes = (*I)->GetIndexFiles();
 440       for (vector<pkgIndexFile *>::const_iterator J = Indexes->begin();
 441            J != Indexes->end(); ++J)
 442       {
 443          if ((*J)->FindInCache(*File.Cache()) == File)
 444          {
 445             Found = (*J);
 446             return true;
 447          }
 448       }
 449    }
 450
 451    return false;
 452 }
 453                                                                         /*}}}*/
 454 // SourceList::GetIndexes - Load the index files into the downloader    /*{{{*/
 455 // ---------------------------------------------------------------------
 456 /* */
 457 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 458 {
 459    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 460       if ((*I)->GetIndexes(Owner,GetAll) == false)
 461          return false;
 462    return true;
 463 }
 464                                                                         /*}}}*/
 465 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
 466 // SourceList::ReadSourceDir - Read a directory with sources files
 467 // Based on ReadConfigDir()                                             /*{{{*/
 468 // ---------------------------------------------------------------------
 469 /* */
 470 bool pkgSourceList::ReadSourceDir(string Dir)
 471 {
 472    vector<string> const List = GetListOfFilesInDir(Dir, "list", true);
 473
 474    // Read the files
 475    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 476       if (ReadAppend(*I) == false)
 477          return false;
 478    return true;
 479
 480 }
 481                                                                         /*}}}*/
 482 // GetLastModified()                                            /*{{{*/
 483 // ---------------------------------------------------------------------
 484 /* */
 485 time_t pkgSourceList::GetLastModifiedTime()
 486 {
 487    vector<string> List;
 488
 489    string Main = _config->FindFile("Dir::Etc::sourcelist");
 490    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 491
 492    // go over the parts
 493    if (DirectoryExists(Parts) == true)
 494       List = GetListOfFilesInDir(Parts, "list", true);
 495
 496    // calculate the time
 497    time_t mtime_sources = GetModificationTime(Main);
 498    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 499       mtime_sources = std::max(mtime_sources, GetModificationTime(*I));
 500
 501    return mtime_sources;
 502 }
 503                                                                         /*}}}*/
 504