apt-pkg/sourcelist.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
   4 /* ######################################################################
   5
   6    List of Sources
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include<config.h>
  12
  13 #include <apt-pkg/sourcelist.h>
  14 #include <apt-pkg/error.h>
  15 #include <apt-pkg/fileutl.h>
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/metaindex.h>
  19 #include <apt-pkg/indexfile.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/pkgcache.h>
  22 #include <apt-pkg/cacheiterators.h>
  23
  24 #include <ctype.h>
  25 #include <stddef.h>
  26 #include <time.h>
  27 #include <cstring>
  28 #include <map>
  29 #include <string>
  30 #include <vector>
  31 #include <fstream>
  32 #include <algorithm>
  33
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39 // Global list of Items supported
  40 static pkgSourceList::Type *ItmList[10];
  41 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
  42 unsigned long pkgSourceList::Type::GlobalListLen = 0;
  43
  44 // Type::Type - Constructor                                             /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* Link this to the global list of items*/
  47 pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
  48 {
  49    ItmList[GlobalListLen] = this;
  50    ++GlobalListLen;
  51 }
  52 pkgSourceList::Type::~Type() {}
  53                                                                         /*}}}*/
  54 // Type::GetType - Get a specific meta for a given type                 /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
  58 {
  59    for (unsigned I = 0; I != GlobalListLen; ++I)
  60       if (strcmp(GlobalList[I]->Name,Type) == 0)
  61          return GlobalList[I];
  62    return 0;
  63 }
  64                                                                         /*}}}*/
  65 // Type::FixupURI - Normalize the URI and check it..                    /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* */
  68 bool pkgSourceList::Type::FixupURI(string &URI) const
  69 {
  70    if (URI.empty() == true)
  71       return false;
  72
  73    if (URI.find(':') == string::npos)
  74       return false;
  75
  76    URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
  77
  78    // Make sure that the URI is / postfixed
  79    if (URI[URI.size() - 1] != '/')
  80       URI += '/';
  81
  82    return true;
  83 }
  84                                                                         /*}}}*/
  85 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,        /*{{{*/
  86                                       pkgTagSection &Tags,
  87                                       unsigned int const i,
  88                                       FileFd &Fd)
  89 {
  90    map<string, string> Options;
  91
  92    string Enabled = Tags.FindS("Enabled");
  93    if (Enabled.empty() == false && StringToBool(Enabled) == false)
  94       return true;
  95
  96    std::map<char const * const, std::pair<char const * const, bool> > mapping;
  97 #define APT_PLUSMINUS(X, Y) \
  98    mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \
  99    mapping.insert(std::make_pair(X "Add", std::make_pair(Y "+", true))); \
 100    mapping.insert(std::make_pair(X "Remove", std::make_pair(Y "-", true)))
 101    APT_PLUSMINUS("Architectures", "arch");
 102    APT_PLUSMINUS("Languages", "lang");
 103    APT_PLUSMINUS("Targets", "target");
 104 #undef APT_PLUSMINUS
 105    mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false)));
 106    mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false)));
 107    mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false)));
 108    mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
 109    mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
 110    mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
 111    mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));
 112
 113    for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
 114       if (Tags.Exists(m->first))
 115       {
 116          std::string option = Tags.FindS(m->first);
 117          // for deb822 the " " is the delimiter, but the backend expects ","
 118          if (m->second.second == true)
 119             std::replace(option.begin(), option.end(), ' ', ',');
 120          Options[m->second.first] = option;
 121       }
 122
 123    {
 124       std::string entry;
 125       strprintf(entry, "%s:%i", Fd.Name().c_str(), i);
 126       Options["sourceslist-entry"] = entry;
 127    }
 128
 129    // now create one item per suite/section
 130    string Suite = Tags.FindS("Suites");
 131    Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
 132    string const Component = Tags.FindS("Components");
 133    string const URIS = Tags.FindS("URIs");
 134
 135    std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
 136    std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
 137    std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
 138
 139    if (list_uris.empty())
 140       // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
 141       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
 142
 143    for (std::vector<std::string>::const_iterator U = list_uris.begin();
 144         U != list_uris.end(); ++U)
 145    {
 146       std::string URI = *U;
 147       if (U->empty() || FixupURI(URI) == false)
 148          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
 149
 150       if (list_suite.empty())
 151          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
 152
 153       for (std::vector<std::string>::const_iterator S = list_suite.begin();
 154            S != list_suite.end(); ++S)
 155       {
 156          if (S->empty() == false && (*S)[S->size() - 1] == '/')
 157          {
 158             if (list_comp.empty() == false)
 159                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
 160             if (CreateItem(List, URI, *S, "", Options) == false)
 161                return false;
 162          }
 163          else
 164          {
 165             if (list_comp.empty())
 166                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
 167
 168             for (std::vector<std::string>::const_iterator C = list_comp.begin();
 169                   C != list_comp.end(); ++C)
 170             {
 171                if (CreateItem(List, URI, *S, *C, Options) == false)
 172                {
 173                   return false;
 174                }
 175             }
 176          }
 177       }
 178    }
 179    return true;
 180 }
 181                                                                         /*}}}*/
 182 // Type::ParseLine - Parse a single line                                /*{{{*/
 183 // ---------------------------------------------------------------------
 184 /* This is a generic one that is the 'usual' format for sources.list
 185    Weird types may override this. */
 186 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 187                                     const char *Buffer,
 188                                     unsigned int const CurLine,
 189                                     string const &File) const
 190 {
 191    for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 192
 193    // Parse option field if it exists
 194    // e.g.: [ option1=value1 option2=value2 ]
 195    map<string, string> Options;
 196    {
 197       std::string entry;
 198       strprintf(entry, "%s:%i", File.c_str(), CurLine);
 199       Options["sourceslist-entry"] = entry;
 200    }
 201    if (Buffer != 0 && Buffer[0] == '[')
 202    {
 203       ++Buffer; // ignore the [
 204       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 205       while (*Buffer != ']')
 206       {
 207          // get one option, e.g. option1=value1
 208          string option;
 209          if (ParseQuoteWord(Buffer,option) == false)
 210             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
 211
 212          if (option.length() < 3)
 213             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
 214
 215          // accept options even if the last has no space before the ]-end marker
 216          if (option.at(option.length()-1) == ']')
 217          {
 218             for (; *Buffer != ']'; --Buffer);
 219             option.resize(option.length()-1);
 220          }
 221
 222          size_t const needle = option.find('=');
 223          if (needle == string::npos)
 224             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
 225
 226          string const key = string(option, 0, needle);
 227          string const value = string(option, needle + 1, option.length());
 228
 229          if (key.empty() == true)
 230             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
 231
 232          if (value.empty() == true)
 233             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
 234
 235          Options[key] = value;
 236       }
 237       ++Buffer; // ignore the ]
 238       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 239    }
 240
 241    string URI;
 242    string Dist;
 243    string Section;
 244
 245    if (ParseQuoteWord(Buffer,URI) == false)
 246       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
 247    if (ParseQuoteWord(Buffer,Dist) == false)
 248       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
 249
 250    if (FixupURI(URI) == false)
 251       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
 252
 253    // Check for an absolute dists specification.
 254    if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
 255    {
 256       if (ParseQuoteWord(Buffer,Section) == true)
 257          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
 258       Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
 259       return CreateItem(List, URI, Dist, Section, Options);
 260    }
 261
 262    // Grab the rest of the dists
 263    if (ParseQuoteWord(Buffer,Section) == false)
 264       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
 265
 266    do
 267    {
 268       if (CreateItem(List, URI, Dist, Section, Options) == false)
 269          return false;
 270    }
 271    while (ParseQuoteWord(Buffer,Section) == true);
 272
 273    return true;
 274 }
 275                                                                         /*}}}*/
 276 // SourceList::pkgSourceList - Constructors                             /*{{{*/
 277 // ---------------------------------------------------------------------
 278 /* */
 279 pkgSourceList::pkgSourceList() : d(NULL)
 280 {
 281 }
 282                                                                         /*}}}*/
 283 // SourceList::~pkgSourceList - Destructor                              /*{{{*/
 284 // ---------------------------------------------------------------------
 285 /* */
 286 pkgSourceList::~pkgSourceList()
 287 {
 288    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 289       delete *I;
 290    SrcList.clear();
 291    for (auto  F = VolatileFiles.begin(); F != VolatileFiles.end(); ++F)
 292       delete (*F);
 293    VolatileFiles.clear();
 294 }
 295                                                                         /*}}}*/
 296 // SourceList::ReadMainList - Read the main source list from etc        /*{{{*/
 297 // ---------------------------------------------------------------------
 298 /* */
 299 bool pkgSourceList::ReadMainList()
 300 {
 301    // CNC:2003-03-03 - Multiple sources list support.
 302    bool Res = true;
 303 #if 0
 304    Res = ReadVendors();
 305    if (Res == false)
 306       return false;
 307 #endif
 308
 309    Reset();
 310    // CNC:2003-11-28 - Entries in sources.list have priority over
 311    //                  entries in sources.list.d.
 312    string Main = _config->FindFile("Dir::Etc::sourcelist");
 313    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 314
 315    if (RealFileExists(Main) == true)
 316       Res &= ReadAppend(Main);
 317    else if (DirectoryExists(Parts) == false)
 318       // Only warn if there are no sources.list.d.
 319       _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
 320
 321    if (DirectoryExists(Parts) == true)
 322       Res &= ReadSourceDir(Parts);
 323    else if (RealFileExists(Main) == false)
 324       // Only warn if there is no sources.list file.
 325       _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
 326
 327    return Res;
 328 }
 329                                                                         /*}}}*/
 330 // SourceList::Reset - Clear the sourcelist contents                    /*{{{*/
 331 // ---------------------------------------------------------------------
 332 /* */
 333 void pkgSourceList::Reset()
 334 {
 335    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 336       delete *I;
 337    SrcList.clear();
 338 }
 339                                                                         /*}}}*/
 340 // SourceList::Read - Parse the sourcelist file                         /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* */
 343 bool pkgSourceList::Read(string const &File)
 344 {
 345    Reset();
 346    return ReadAppend(File);
 347 }
 348                                                                         /*}}}*/
 349 // SourceList::ReadAppend - Parse a sourcelist file                     /*{{{*/
 350 // ---------------------------------------------------------------------
 351 /* */
 352 bool pkgSourceList::ReadAppend(string const &File)
 353 {
 354    if (flExtension(File) == "sources")
 355       return ParseFileDeb822(File);
 356    else
 357       return ParseFileOldStyle(File);
 358 }
 359                                                                         /*}}}*/
 360 // SourceList::ReadFileOldStyle - Read Traditional style sources.list   /*{{{*/
 361 // ---------------------------------------------------------------------
 362 /* */
 363 bool pkgSourceList::ParseFileOldStyle(std::string const &File)
 364 {
 365    // Open the stream for reading
 366    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
 367    if (F.fail() == true)
 368       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
 369
 370    std::string Buffer;
 371    for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
 372    {
 373       // remove comments
 374       size_t curpos = 0;
 375       while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
 376       {
 377          size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
 378          size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
 379          if (openbrackets > closedbrackets)
 380          {
 381             // a # in an option, unlikely, but oh well, it was supported so stick to it
 382             ++curpos;
 383             continue;
 384          }
 385          Buffer.erase(curpos);
 386          break;
 387       }
 388       // remove spaces before/after
 389       curpos = Buffer.find_first_not_of(" \t\r");
 390       if (curpos != 0)
 391          Buffer.erase(0, curpos);
 392       curpos = Buffer.find_last_not_of(" \t\r");
 393       if (curpos != std::string::npos)
 394          Buffer.erase(curpos + 1);
 395
 396       if (Buffer.empty())
 397          continue;
 398
 399       // Grok it
 400       std::string const LineType = Buffer.substr(0, Buffer.find_first_of(" \t\v"));
 401       if (LineType.empty() || LineType == Buffer)
 402          return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
 403
 404       Type *Parse = Type::GetType(LineType.c_str());
 405       if (Parse == 0)
 406          return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
 407
 408       if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
 409          return false;
 410    }
 411    return true;
 412 }
 413                                                                         /*}}}*/
 414 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list        /*{{{*/
 415 // ---------------------------------------------------------------------
 416 /* Returns: the number of stanzas parsed*/
 417 bool pkgSourceList::ParseFileDeb822(string const &File)
 418 {
 419    pkgUserTagSection Tags;
 420    unsigned int i = 1;
 421
 422    // see if we can read the file
 423    FileFd Fd(File, FileFd::ReadOnly);
 424    pkgTagFile Sources(&Fd);
 425    if (Fd.IsOpen() == false || Fd.Failed())
 426       return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 427
 428    // read step by step
 429    while (Sources.Step(Tags) == true)
 430    {
 431       if(Tags.Exists("Types") == false)
 432          return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 433
 434       string const types = Tags.FindS("Types");
 435       std::vector<std::string> const list_types = VectorizeString(types, ' ');
 436       for (std::vector<std::string>::const_iterator I = list_types.begin();
 437         I != list_types.end(); ++I)
 438       {
 439          Type *Parse = Type::GetType((*I).c_str());
 440          if (Parse == 0)
 441          {
 442             _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
 443             return false;
 444          }
 445
 446          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
 447             return false;
 448
 449          ++i;
 450       }
 451    }
 452    return true;
 453 }
 454                                                                         /*}}}*/
 455 // SourceList::FindIndex - Get the index associated with a file         /*{{{*/
 456 static bool FindInIndexFileContainer(std::vector<pkgIndexFile *> const &Cont, pkgCache::PkgFileIterator const &File, pkgIndexFile *&Found)
 457 {
 458    auto const J = std::find_if(Cont.begin(), Cont.end(), [&File](pkgIndexFile const * const J) {
 459          return J->FindInCache(*File.Cache()) == File;
 460    });
 461    if (J != Cont.end())
 462    {
 463       Found = (*J);
 464       return true;
 465    }
 466    return false;
 467 }
 468 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
 469                               pkgIndexFile *&Found) const
 470 {
 471    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 472       if (FindInIndexFileContainer(*(*I)->GetIndexFiles(), File, Found))
 473          return true;
 474
 475    return FindInIndexFileContainer(VolatileFiles, File, Found);
 476 }
 477                                                                         /*}}}*/
 478 // SourceList::GetIndexes - Load the index files into the downloader    /*{{{*/
 479 // ---------------------------------------------------------------------
 480 /* */
 481 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 482 {
 483    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 484       if ((*I)->GetIndexes(Owner,GetAll) == false)
 485          return false;
 486    return true;
 487 }
 488                                                                         /*}}}*/
 489 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
 490 // SourceList::ReadSourceDir - Read a directory with sources files
 491 // Based on ReadConfigDir()                                             /*{{{*/
 492 // ---------------------------------------------------------------------
 493 /* */
 494 bool pkgSourceList::ReadSourceDir(string const &Dir)
 495 {
 496    std::vector<std::string> ext;
 497    ext.push_back("list");
 498    ext.push_back("sources");
 499    std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
 500
 501    // Read the files
 502    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 503       if (ReadAppend(*I) == false)
 504          return false;
 505    return true;
 506
 507 }
 508                                                                         /*}}}*/
 509 // GetLastModified()                                            /*{{{*/
 510 // ---------------------------------------------------------------------
 511 /* */
 512 time_t pkgSourceList::GetLastModifiedTime()
 513 {
 514    vector<string> List;
 515
 516    string Main = _config->FindFile("Dir::Etc::sourcelist");
 517    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 518
 519    // go over the parts
 520    if (DirectoryExists(Parts) == true)
 521       List = GetListOfFilesInDir(Parts, "list", true);
 522
 523    // calculate the time
 524    std::vector<time_t> modtimes;
 525    modtimes.reserve(1 + List.size());
 526    modtimes.push_back(GetModificationTime(Main));
 527    std::transform(List.begin(), List.end(), std::back_inserter(modtimes), GetModificationTime);
 528    auto const maxmtime = std::max_element(modtimes.begin(), modtimes.end());
 529    return *maxmtime;
 530 }
 531                                                                         /*}}}*/
 532 std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const      /*{{{*/
 533 {
 534    return VolatileFiles;
 535 }
 536                                                                         /*}}}*/
 537 void pkgSourceList::AddVolatileFile(pkgIndexFile * const File)          /*{{{*/
 538 {
 539    if (File != NULL)
 540       VolatileFiles.push_back(File);
 541 }
 542                                                                         /*}}}*/