apt-pkg/sourcelist.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
   4 /* ######################################################################
   5
   6    List of Sources
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include<config.h>
  12
  13 #include <apt-pkg/sourcelist.h>
  14 #include <apt-pkg/cmndline.h>
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/fileutl.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/configuration.h>
  19 #include <apt-pkg/metaindex.h>
  20 #include <apt-pkg/indexfile.h>
  21 #include <apt-pkg/tagfile.h>
  22 #include <apt-pkg/pkgcache.h>
  23 #include <apt-pkg/cacheiterators.h>
  24 #include <apt-pkg/debindexfile.h>
  25
  26 #include <ctype.h>
  27 #include <stddef.h>
  28 #include <time.h>
  29 #include <cstring>
  30 #include <map>
  31 #include <string>
  32 #include <vector>
  33 #include <fstream>
  34 #include <algorithm>
  35
  36 #include <apti18n.h>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Global list of Items supported
  42 static pkgSourceList::Type *ItmList[10];
  43 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
  44 unsigned long pkgSourceList::Type::GlobalListLen = 0;
  45
  46 // Type::Type - Constructor                                             /*{{{*/
  47 // ---------------------------------------------------------------------
  48 /* Link this to the global list of items*/
  49 pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
  50 {
  51    ItmList[GlobalListLen] = this;
  52    ++GlobalListLen;
  53 }
  54 pkgSourceList::Type::~Type() {}
  55                                                                         /*}}}*/
  56 // Type::GetType - Get a specific meta for a given type                 /*{{{*/
  57 // ---------------------------------------------------------------------
  58 /* */
  59 pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
  60 {
  61    for (unsigned I = 0; I != GlobalListLen; ++I)
  62       if (strcmp(GlobalList[I]->Name,Type) == 0)
  63          return GlobalList[I];
  64    return 0;
  65 }
  66                                                                         /*}}}*/
  67 // Type::FixupURI - Normalize the URI and check it..                    /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 bool pkgSourceList::Type::FixupURI(string &URI) const
  71 {
  72    if (URI.empty() == true)
  73       return false;
  74
  75    if (URI.find(':') == string::npos)
  76       return false;
  77
  78    URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
  79
  80    // Make sure that the URI is / postfixed
  81    if (URI[URI.size() - 1] != '/')
  82       URI += '/';
  83
  84    return true;
  85 }
  86                                                                         /*}}}*/
  87 bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,        /*{{{*/
  88                                       pkgTagSection &Tags,
  89                                       unsigned int const i,
  90                                       FileFd &Fd)
  91 {
  92    map<string, string> Options;
  93
  94    string Enabled = Tags.FindS("Enabled");
  95    if (Enabled.empty() == false && StringToBool(Enabled) == false)
  96       return true;
  97
  98    std::map<char const * const, std::pair<char const * const, bool> > mapping;
  99 #define APT_PLUSMINUS(X, Y) \
 100    mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \
 101    mapping.insert(std::make_pair(X "Add", std::make_pair(Y "+", true))); \
 102    mapping.insert(std::make_pair(X "Remove", std::make_pair(Y "-", true)))
 103    APT_PLUSMINUS("Architectures", "arch");
 104    APT_PLUSMINUS("Languages", "lang");
 105    APT_PLUSMINUS("Targets", "target");
 106 #undef APT_PLUSMINUS
 107    mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false)));
 108    mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false)));
 109    mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false)));
 110    mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
 111    mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
 112    mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
 113    mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));
 114
 115    for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
 116       if (Tags.Exists(m->first))
 117       {
 118          std::string option = Tags.FindS(m->first);
 119          // for deb822 the " " is the delimiter, but the backend expects ","
 120          if (m->second.second == true)
 121             std::replace(option.begin(), option.end(), ' ', ',');
 122          Options[m->second.first] = option;
 123       }
 124
 125    {
 126       std::string entry;
 127       strprintf(entry, "%s:%i", Fd.Name().c_str(), i);
 128       Options["sourceslist-entry"] = entry;
 129    }
 130
 131    // now create one item per suite/section
 132    string Suite = Tags.FindS("Suites");
 133    Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
 134    string const Component = Tags.FindS("Components");
 135    string const URIS = Tags.FindS("URIs");
 136
 137    std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
 138    std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
 139    std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
 140
 141    if (list_uris.empty())
 142       // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
 143       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
 144
 145    for (std::vector<std::string>::const_iterator U = list_uris.begin();
 146         U != list_uris.end(); ++U)
 147    {
 148       std::string URI = *U;
 149       if (U->empty() || FixupURI(URI) == false)
 150          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
 151
 152       if (list_suite.empty())
 153          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
 154
 155       for (std::vector<std::string>::const_iterator S = list_suite.begin();
 156            S != list_suite.end(); ++S)
 157       {
 158          if (S->empty() == false && (*S)[S->size() - 1] == '/')
 159          {
 160             if (list_comp.empty() == false)
 161                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
 162             if (CreateItem(List, URI, *S, "", Options) == false)
 163                return false;
 164          }
 165          else
 166          {
 167             if (list_comp.empty())
 168                return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
 169
 170             for (std::vector<std::string>::const_iterator C = list_comp.begin();
 171                   C != list_comp.end(); ++C)
 172             {
 173                if (CreateItem(List, URI, *S, *C, Options) == false)
 174                {
 175                   return false;
 176                }
 177             }
 178          }
 179       }
 180    }
 181    return true;
 182 }
 183                                                                         /*}}}*/
 184 // Type::ParseLine - Parse a single line                                /*{{{*/
 185 // ---------------------------------------------------------------------
 186 /* This is a generic one that is the 'usual' format for sources.list
 187    Weird types may override this. */
 188 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 189                                     const char *Buffer,
 190                                     unsigned int const CurLine,
 191                                     string const &File) const
 192 {
 193    for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 194
 195    // Parse option field if it exists
 196    // e.g.: [ option1=value1 option2=value2 ]
 197    map<string, string> Options;
 198    {
 199       std::string entry;
 200       strprintf(entry, "%s:%i", File.c_str(), CurLine);
 201       Options["sourceslist-entry"] = entry;
 202    }
 203    if (Buffer != 0 && Buffer[0] == '[')
 204    {
 205       ++Buffer; // ignore the [
 206       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 207       while (*Buffer != ']')
 208       {
 209          // get one option, e.g. option1=value1
 210          string option;
 211          if (ParseQuoteWord(Buffer,option) == false)
 212             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
 213
 214          if (option.length() < 3)
 215             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
 216
 217          // accept options even if the last has no space before the ]-end marker
 218          if (option.at(option.length()-1) == ']')
 219          {
 220             for (; *Buffer != ']'; --Buffer);
 221             option.resize(option.length()-1);
 222          }
 223
 224          size_t const needle = option.find('=');
 225          if (needle == string::npos)
 226             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
 227
 228          string const key = string(option, 0, needle);
 229          string const value = string(option, needle + 1, option.length());
 230
 231          if (key.empty() == true)
 232             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
 233
 234          if (value.empty() == true)
 235             return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
 236
 237          Options[key] = value;
 238       }
 239       ++Buffer; // ignore the ]
 240       for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
 241    }
 242
 243    string URI;
 244    string Dist;
 245    string Section;
 246
 247    if (ParseQuoteWord(Buffer,URI) == false)
 248       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
 249    if (ParseQuoteWord(Buffer,Dist) == false)
 250       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
 251
 252    if (FixupURI(URI) == false)
 253       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
 254
 255    // Check for an absolute dists specification.
 256    if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
 257    {
 258       if (ParseQuoteWord(Buffer,Section) == true)
 259          return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
 260       Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
 261       return CreateItem(List, URI, Dist, Section, Options);
 262    }
 263
 264    // Grab the rest of the dists
 265    if (ParseQuoteWord(Buffer,Section) == false)
 266       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
 267
 268    do
 269    {
 270       if (CreateItem(List, URI, Dist, Section, Options) == false)
 271          return false;
 272    }
 273    while (ParseQuoteWord(Buffer,Section) == true);
 274
 275    return true;
 276 }
 277                                                                         /*}}}*/
 278 // SourceList::pkgSourceList - Constructors                             /*{{{*/
 279 // ---------------------------------------------------------------------
 280 /* */
 281 pkgSourceList::pkgSourceList() : d(NULL)
 282 {
 283 }
 284                                                                         /*}}}*/
 285 // SourceList::~pkgSourceList - Destructor                              /*{{{*/
 286 // ---------------------------------------------------------------------
 287 /* */
 288 pkgSourceList::~pkgSourceList()
 289 {
 290    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 291       delete *I;
 292    SrcList.clear();
 293    for (auto  F = VolatileFiles.begin(); F != VolatileFiles.end(); ++F)
 294       delete (*F);
 295    VolatileFiles.clear();
 296 }
 297                                                                         /*}}}*/
 298 // SourceList::ReadMainList - Read the main source list from etc        /*{{{*/
 299 // ---------------------------------------------------------------------
 300 /* */
 301 bool pkgSourceList::ReadMainList()
 302 {
 303    // CNC:2003-03-03 - Multiple sources list support.
 304    bool Res = true;
 305 #if 0
 306    Res = ReadVendors();
 307    if (Res == false)
 308       return false;
 309 #endif
 310
 311    Reset();
 312    // CNC:2003-11-28 - Entries in sources.list have priority over
 313    //                  entries in sources.list.d.
 314    string Main = _config->FindFile("Dir::Etc::sourcelist");
 315    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 316
 317    if (RealFileExists(Main) == true)
 318       Res &= ReadAppend(Main);
 319    else if (DirectoryExists(Parts) == false)
 320       // Only warn if there are no sources.list.d.
 321       _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
 322
 323    if (DirectoryExists(Parts) == true)
 324       Res &= ReadSourceDir(Parts);
 325    else if (RealFileExists(Main) == false)
 326       // Only warn if there is no sources.list file.
 327       _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
 328
 329    return Res;
 330 }
 331                                                                         /*}}}*/
 332 // SourceList::Reset - Clear the sourcelist contents                    /*{{{*/
 333 // ---------------------------------------------------------------------
 334 /* */
 335 void pkgSourceList::Reset()
 336 {
 337    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 338       delete *I;
 339    SrcList.clear();
 340 }
 341                                                                         /*}}}*/
 342 // SourceList::Read - Parse the sourcelist file                         /*{{{*/
 343 // ---------------------------------------------------------------------
 344 /* */
 345 bool pkgSourceList::Read(string const &File)
 346 {
 347    Reset();
 348    return ReadAppend(File);
 349 }
 350                                                                         /*}}}*/
 351 // SourceList::ReadAppend - Parse a sourcelist file                     /*{{{*/
 352 // ---------------------------------------------------------------------
 353 /* */
 354 bool pkgSourceList::ReadAppend(string const &File)
 355 {
 356    if (flExtension(File) == "sources")
 357       return ParseFileDeb822(File);
 358    else
 359       return ParseFileOldStyle(File);
 360 }
 361                                                                         /*}}}*/
 362 // SourceList::ReadFileOldStyle - Read Traditional style sources.list   /*{{{*/
 363 // ---------------------------------------------------------------------
 364 /* */
 365 bool pkgSourceList::ParseFileOldStyle(std::string const &File)
 366 {
 367    // Open the stream for reading
 368    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
 369    if (F.fail() == true)
 370       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
 371
 372    std::string Buffer;
 373    for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
 374    {
 375       // remove comments
 376       size_t curpos = 0;
 377       while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
 378       {
 379          size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
 380          size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
 381          if (openbrackets > closedbrackets)
 382          {
 383             // a # in an option, unlikely, but oh well, it was supported so stick to it
 384             ++curpos;
 385             continue;
 386          }
 387          Buffer.erase(curpos);
 388          break;
 389       }
 390       // remove spaces before/after
 391       curpos = Buffer.find_first_not_of(" \t\r");
 392       if (curpos != 0)
 393          Buffer.erase(0, curpos);
 394       curpos = Buffer.find_last_not_of(" \t\r");
 395       if (curpos != std::string::npos)
 396          Buffer.erase(curpos + 1);
 397
 398       if (Buffer.empty())
 399          continue;
 400
 401       // Grok it
 402       std::string const LineType = Buffer.substr(0, Buffer.find_first_of(" \t\v"));
 403       if (LineType.empty() || LineType == Buffer)
 404          return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
 405
 406       Type *Parse = Type::GetType(LineType.c_str());
 407       if (Parse == 0)
 408          return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
 409
 410       if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
 411          return false;
 412    }
 413    return true;
 414 }
 415                                                                         /*}}}*/
 416 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list        /*{{{*/
 417 // ---------------------------------------------------------------------
 418 /* Returns: the number of stanzas parsed*/
 419 bool pkgSourceList::ParseFileDeb822(string const &File)
 420 {
 421    pkgUserTagSection Tags;
 422    unsigned int i = 1;
 423
 424    // see if we can read the file
 425    FileFd Fd(File, FileFd::ReadOnly);
 426    pkgTagFile Sources(&Fd);
 427    if (Fd.IsOpen() == false || Fd.Failed())
 428       return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 429
 430    // read step by step
 431    while (Sources.Step(Tags) == true)
 432    {
 433       if(Tags.Exists("Types") == false)
 434          return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 435
 436       string const types = Tags.FindS("Types");
 437       std::vector<std::string> const list_types = VectorizeString(types, ' ');
 438       for (std::vector<std::string>::const_iterator I = list_types.begin();
 439         I != list_types.end(); ++I)
 440       {
 441          Type *Parse = Type::GetType((*I).c_str());
 442          if (Parse == 0)
 443          {
 444             _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
 445             return false;
 446          }
 447
 448          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
 449             return false;
 450
 451          ++i;
 452       }
 453    }
 454    return true;
 455 }
 456                                                                         /*}}}*/
 457 // SourceList::FindIndex - Get the index associated with a file         /*{{{*/
 458 static bool FindInIndexFileContainer(std::vector<pkgIndexFile *> const &Cont, pkgCache::PkgFileIterator const &File, pkgIndexFile *&Found)
 459 {
 460    auto const J = std::find_if(Cont.begin(), Cont.end(), [&File](pkgIndexFile const * const J) {
 461          return J->FindInCache(*File.Cache()) == File;
 462    });
 463    if (J != Cont.end())
 464    {
 465       Found = (*J);
 466       return true;
 467    }
 468    return false;
 469 }
 470 bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
 471                               pkgIndexFile *&Found) const
 472 {
 473    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 474       if (FindInIndexFileContainer(*(*I)->GetIndexFiles(), File, Found))
 475          return true;
 476
 477    return FindInIndexFileContainer(VolatileFiles, File, Found);
 478 }
 479                                                                         /*}}}*/
 480 // SourceList::GetIndexes - Load the index files into the downloader    /*{{{*/
 481 // ---------------------------------------------------------------------
 482 /* */
 483 bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 484 {
 485    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
 486       if ((*I)->GetIndexes(Owner,GetAll) == false)
 487          return false;
 488    return true;
 489 }
 490                                                                         /*}}}*/
 491 // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
 492 // SourceList::ReadSourceDir - Read a directory with sources files
 493 // Based on ReadConfigDir()                                             /*{{{*/
 494 // ---------------------------------------------------------------------
 495 /* */
 496 bool pkgSourceList::ReadSourceDir(string const &Dir)
 497 {
 498    std::vector<std::string> ext;
 499    ext.push_back("list");
 500    ext.push_back("sources");
 501    std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
 502
 503    // Read the files
 504    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
 505       if (ReadAppend(*I) == false)
 506          return false;
 507    return true;
 508
 509 }
 510                                                                         /*}}}*/
 511 // GetLastModified()                                            /*{{{*/
 512 // ---------------------------------------------------------------------
 513 /* */
 514 time_t pkgSourceList::GetLastModifiedTime()
 515 {
 516    vector<string> List;
 517
 518    string Main = _config->FindFile("Dir::Etc::sourcelist");
 519    string Parts = _config->FindDir("Dir::Etc::sourceparts");
 520
 521    // go over the parts
 522    if (DirectoryExists(Parts) == true)
 523       List = GetListOfFilesInDir(Parts, "list", true);
 524
 525    // calculate the time
 526    std::vector<time_t> modtimes;
 527    modtimes.reserve(1 + List.size());
 528    modtimes.push_back(GetModificationTime(Main));
 529    std::transform(List.begin(), List.end(), std::back_inserter(modtimes), GetModificationTime);
 530    auto const maxmtime = std::max_element(modtimes.begin(), modtimes.end());
 531    return *maxmtime;
 532 }
 533                                                                         /*}}}*/
 534 std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const      /*{{{*/
 535 {
 536    return VolatileFiles;
 537 }
 538                                                                         /*}}}*/
 539 void pkgSourceList::AddVolatileFile(pkgIndexFile * const File)          /*{{{*/
 540 {
 541    if (File != nullptr)
 542       VolatileFiles.push_back(File);
 543 }
 544                                                                         /*}}}*/
 545 bool pkgSourceList::AddVolatileFile(std::string const &File)            /*{{{*/
 546 {
 547    if (File.empty() || FileExists(File) == false)
 548       return false;
 549
 550    if (flExtension(File) == "deb")
 551       AddVolatileFile(new debDebPkgFileIndex(File));
 552    else
 553       return false;
 554
 555    return true;
 556 }
 557                                                                         /*}}}*/
 558 void pkgSourceList::AddVolatileFiles(CommandLine &CmdL, std::vector<const char*> * const VolatileCmdL)/*{{{*/
 559 {
 560    std::remove_if(CmdL.FileList + 1, CmdL.FileList + 1 + CmdL.FileSize(), [&](char const * const I) {
 561       if (I != nullptr && (I[0] == '/' || (I[0] == '.' && I[1] == '/')))
 562       {
 563          if (AddVolatileFile(I))
 564          {
 565             if (VolatileCmdL != nullptr)
 566                VolatileCmdL->push_back(I);
 567          }
 568          else
 569             _error->Error(_("Unsupported file %s given on commandline"), I);
 570          return true;
 571       }
 572       return false;
 573    });
 574 }
 575                                                                         /*}}}*/