ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/deblistparser.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/fileutl.h>
  20 #include <apt-pkg/gpgv.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/debfile.h>
  25 #include <apt-pkg/pkgcache.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/sha2.h>
  28 #include <apt-pkg/tagfile.h>
  29
  30 #include <ctype.h>
  31 #include <fnmatch.h>
  32 #include <ftw.h>
  33 #include <locale.h>
  34 #include <string.h>
  35 #include <sys/stat.h>
  36 #include <sys/types.h>
  37 #include <unistd.h>
  38 #include <ctime>
  39 #include <iostream>
  40 #include <sstream>
  41 #include <memory>
  42 #include <utility>
  43
  44 #include "apt-ftparchive.h"
  45 #include "writer.h"
  46 #include "cachedb.h"
  47 #include "multicompress.h"
  48
  49 #include <apti18n.h>
  50                                                                         /*}}}*/
  51 using namespace std;
  52 FTWScanner *FTWScanner::Owner;
  53
  54 // ConfigToDoHashes - which hashes to generate                          /*{{{*/
  55 static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
  56 {
  57    if (_config->FindB(Conf, true) == true)
  58       DoHashes |= Flag;
  59    else
  60       DoHashes &= ~Flag;
  61 }
  62 static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
  63 {
  64    SingleConfigToDoHashes(DoHashes, Conf + "::MD5", Hashes::MD5SUM);
  65    SingleConfigToDoHashes(DoHashes, Conf + "::SHA1", Hashes::SHA1SUM);
  66    SingleConfigToDoHashes(DoHashes, Conf + "::SHA256", Hashes::SHA256SUM);
  67    SingleConfigToDoHashes(DoHashes, Conf + "::SHA512", Hashes::SHA512SUM);
  68 }
  69                                                                         /*}}}*/
  70
  71 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  72 FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch): Arch(Arch), DoHashes(~0)
  73 {
  74    if (GivenOutput == NULL)
  75    {
  76       Output = new FileFd;
  77       OwnsOutput = true;
  78       Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
  79    }
  80    else
  81    {
  82       Output = GivenOutput;
  83       OwnsOutput = false;
  84    }
  85    ErrorPrinted = false;
  86    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  87    ConfigToDoHashes(DoHashes, "APT::FTPArchive");
  88 }
  89                                                                         /*}}}*/
  90 FTWScanner::~FTWScanner()
  91 {
  92    if (Output != NULL && OwnsOutput)
  93       delete Output;
  94 }
  95 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* This is the FTW scanner, it processes each directory element in the
  98    directory tree. */
  99 int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
 100 {
 101    if (Flag == FTW_DNR)
 102    {
 103       Owner->NewLine(1);
 104       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
 105    }
 106    if (Flag == FTW_NS)
 107    {
 108       Owner->NewLine(1);
 109       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
 110    }
 111    if (Flag != FTW_F)
 112       return 0;
 113
 114    return ScannerFile(File, true);
 115 }
 116                                                                         /*}}}*/
 117 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
 121 {
 122    const char *LastComponent = strrchr(File, '/');
 123    char *RealPath = NULL;
 124
 125    if (LastComponent == NULL)
 126       LastComponent = File;
 127    else
 128       LastComponent++;
 129
 130    vector<string>::const_iterator I;
 131    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 132    {
 133       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 134          break;
 135    }
 136    if (I == Owner->Patterns.end())
 137       return 0;
 138
 139    /* Process it. If the file is a link then resolve it into an absolute
 140       name.. This works best if the directory components the scanner are
 141       given are not links themselves. */
 142    char Jnk[2];
 143    Owner->OriginalPath = File;
 144    if (ReadLink &&
 145        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 146        (RealPath = realpath(File,NULL)) != 0)
 147    {
 148       Owner->DoPackage(RealPath);
 149       free(RealPath);
 150    }
 151    else
 152       Owner->DoPackage(File);
 153
 154    if (_error->empty() == false)
 155    {
 156       // Print any errors or warnings found
 157       string Err;
 158       bool SeenPath = false;
 159       while (_error->empty() == false)
 160       {
 161          Owner->NewLine(1);
 162
 163          bool const Type = _error->PopMessage(Err);
 164          if (Type == true)
 165             cerr << _("E: ") << Err << endl;
 166          else
 167             cerr << _("W: ") << Err << endl;
 168
 169          if (Err.find(File) != string::npos)
 170             SeenPath = true;
 171       }
 172
 173       if (SeenPath == false)
 174          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 175       return 0;
 176    }
 177
 178    return 0;
 179 }
 180                                                                         /*}}}*/
 181 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 182 // ---------------------------------------------------------------------
 183 /* */
 184 bool FTWScanner::RecursiveScan(string const &Dir)
 185 {
 186    char *RealPath = NULL;
 187    /* If noprefix is set then jam the scan root in, so we don't generate
 188       link followed paths out of control */
 189    if (InternalPrefix.empty() == true)
 190    {
 191       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 192          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 193       InternalPrefix = RealPath;
 194       free(RealPath);
 195    }
 196
 197    // Do recursive directory searching
 198    Owner = this;
 199    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 200
 201    // Error treewalking?
 202    if (Res != 0)
 203    {
 204       if (_error->PendingError() == false)
 205          _error->Errno("ftw",_("Tree walking failed"));
 206       return false;
 207    }
 208
 209    return true;
 210 }
 211                                                                         /*}}}*/
 212 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 213 // ---------------------------------------------------------------------
 214 /* This is an alternative to using FTW to locate files, it reads the list
 215    of files from another file. */
 216 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 217 {
 218    char *RealPath = NULL;
 219    /* If noprefix is set then jam the scan root in, so we don't generate
 220       link followed paths out of control */
 221    if (InternalPrefix.empty() == true)
 222    {
 223       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 224          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 225       InternalPrefix = RealPath;
 226       free(RealPath);
 227    }
 228
 229    Owner = this;
 230    FILE *List = fopen(File.c_str(),"r");
 231    if (List == 0)
 232       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 233
 234    /* We are a tad tricky here.. We prefix the buffer with the directory
 235       name, that way if we need a full path with just use line.. Sneaky and
 236       fully evil. */
 237    char Line[1000];
 238    char *FileStart;
 239    if (Dir.empty() == true || Dir.end()[-1] != '/')
 240       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 241    else
 242       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 243    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 244    {
 245       char *FileName = _strstrip(FileStart);
 246       if (FileName[0] == 0)
 247          continue;
 248
 249       if (FileName[0] != '/')
 250       {
 251          if (FileName != FileStart)
 252             memmove(FileStart,FileName,strlen(FileStart));
 253          FileName = Line;
 254       }
 255
 256 #if 0
 257       struct stat St;
 258       int Flag = FTW_F;
 259       if (stat(FileName,&St) != 0)
 260          Flag = FTW_NS;
 261 #endif
 262
 263       if (ScannerFile(FileName, false) != 0)
 264          break;
 265    }
 266
 267    fclose(List);
 268    return true;
 269 }
 270                                                                         /*}}}*/
 271 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 272 // ---------------------------------------------------------------------
 273 /* */
 274 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 275                         unsigned long long &DeLinkBytes,
 276                         unsigned long long const &FileSize)
 277 {
 278    // See if this isn't an internaly prefix'd file name.
 279    if (InternalPrefix.empty() == false &&
 280        InternalPrefix.length() < FileName.length() &&
 281        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 282                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 283    {
 284       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 285       {
 286          // Tidy up the display
 287          if (DeLinkBytes == 0)
 288             cout << endl;
 289
 290          NewLine(1);
 291          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 292                     SizeToStr(FileSize).c_str());
 293          c1out << flush;
 294
 295          if (NoLinkAct == false)
 296          {
 297             char OldLink[400];
 298             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 299                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 300             else
 301             {
 302                if (unlink(OriginalPath) != 0)
 303                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 304                else
 305                {
 306                   if (link(FileName.c_str(),OriginalPath) != 0)
 307                   {
 308                      // Panic! Restore the symlink
 309                      if (symlink(OldLink,OriginalPath) != 0)
 310                         _error->Errno("symlink", "failed to restore symlink");
 311                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 312                                           FileName.c_str(),
 313                                           OriginalPath);
 314                   }
 315                }
 316             }
 317          }
 318
 319          DeLinkBytes += FileSize;
 320          if (DeLinkBytes/1024 >= DeLinkLimit)
 321             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 322       }
 323
 324       FileName = OriginalPath;
 325    }
 326
 327    return true;
 328 }
 329                                                                         /*}}}*/
 330
 331 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* */
 334 PackagesWriter::PackagesWriter(FileFd * const GivenOutput, TranslationWriter * const transWriter,
 335       string const &DB,string const &Overrides,string const &ExtOverrides,
 336       string const &Arch) :
 337    FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats), TransWriter(transWriter)
 338 {
 339    SetExts(".deb .udeb");
 340    DeLinkLimit = 0;
 341
 342    // Process the command line options
 343    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Packages");
 344    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 345    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 346    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 347    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 348
 349    if (Db.Loaded() == false)
 350       DoContents = false;
 351
 352    // Read the override file
 353    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 354       return;
 355    else
 356       NoOverride = true;
 357
 358    if (ExtOverrides.empty() == false)
 359       Over.ReadExtraOverride(ExtOverrides);
 360
 361    _error->DumpErrors();
 362 }
 363                                                                         /*}}}*/
 364 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* */
 367 bool FTWScanner::SetExts(string const &Vals)
 368 {
 369    ClearPatterns();
 370    string::size_type Start = 0;
 371    while (Start <= Vals.length()-1)
 372    {
 373       string::size_type const Space = Vals.find(' ',Start);
 374       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 375       if ( Arch.empty() == false )
 376       {
 377          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 378          AddPattern(string("*_all") + Vals.substr(Start, Length));
 379       }
 380       else
 381          AddPattern(string("*") + Vals.substr(Start, Length));
 382
 383       Start += Length + 1;
 384    }
 385
 386    return true;
 387 }
 388                                                                         /*}}}*/
 389 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* This method takes a package and gets its control information and
 392    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 393    rewritten and the path/size/hash appended. */
 394 bool PackagesWriter::DoPackage(string FileName)
 395 {
 396    // Pull all the data we need form the DB
 397    if (Db.GetFileInfo(FileName,
 398             true, /* DoControl */
 399             DoContents,
 400             true, /* GenContentsOnly */
 401             false, /* DoSource */
 402             DoHashes, DoAlwaysStat) == false)
 403    {
 404      return false;
 405    }
 406
 407    unsigned long long FileSize = Db.GetFileSize();
 408    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 409       return false;
 410
 411    // Lookup the overide information
 412    pkgTagSection &Tags = Db.Control.Section;
 413    string Package = Tags.FindS("Package");
 414    string Architecture;
 415    // if we generate a Packages file for a given arch, we use it to
 416    // look for overrides. if we run in "simple" mode without the
 417    // "Architecures" variable in the config we use the architecure value
 418    // from the deb file
 419    if(Arch != "")
 420       Architecture = Arch;
 421    else
 422       Architecture = Tags.FindS("Architecture");
 423    unique_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 424
 425    if (Package.empty() == true)
 426       return _error->Error(_("Archive had no package field"));
 427
 428    // If we need to do any rewriting of the header do it now..
 429    if (OverItem.get() == 0)
 430    {
 431       if (NoOverride == false)
 432       {
 433          NewLine(1);
 434          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 435       }
 436
 437       OverItem = unique_ptr<Override::Item>(new Override::Item);
 438       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 439       OverItem->Priority = Tags.FindS("Priority");
 440    }
 441
 442    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 443    string NewFileName;
 444    if (DirStrip.empty() == false &&
 445        FileName.length() > DirStrip.length() &&
 446        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 447                  DirStrip.begin(),DirStrip.end()) == 0)
 448       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 449    else
 450       NewFileName = FileName;
 451    if (PathPrefix.empty() == false)
 452       NewFileName = flCombine(PathPrefix,NewFileName);
 453
 454    /* Configuration says we don't want to include the long Description
 455       in the package file - instead we want to ship a separated file */
 456    string desc;
 457    if (LongDescription == false) {
 458       desc = Tags.FindS("Description").append("\n");
 459       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 460    }
 461
 462    // This lists all the changes to the fields we are going to make.
 463    std::vector<pkgTagSection::Tag> Changes;
 464
 465    std::string Size;
 466    strprintf(Size, "%llu", (unsigned long long) FileSize);
 467    Changes.push_back(pkgTagSection::Tag::Rewrite("Size", Size));
 468
 469    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 470    {
 471       if (hs->HashType() == "MD5Sum")
 472          Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue()));
 473       else if (hs->HashType() == "Checksum-FileSize")
 474          continue;
 475       else
 476          Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue()));
 477    }
 478    Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName));
 479    Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority));
 480    Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 481    Changes.push_back(pkgTagSection::Tag::Remove("Optional"));
 482
 483    string DescriptionMd5;
 484    if (LongDescription == false) {
 485       MD5Summation descmd5;
 486       descmd5.Add(desc.c_str());
 487       DescriptionMd5 = descmd5.Result().Value();
 488       Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5));
 489       if (TransWriter != NULL)
 490          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 491    }
 492
 493    // Rewrite the maintainer field if necessary
 494    bool MaintFailed;
 495    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 496    if (MaintFailed == true)
 497    {
 498       if (NoOverride == false)
 499       {
 500          NewLine(1);
 501          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 502                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 503       }
 504    }
 505
 506    if (NewMaint.empty() == false)
 507       Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint));
 508
 509    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 510       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 511       but dpkg does this append bit. So we do the append bit, at least that way the
 512       status file and package file will remain similar. There are other transforms
 513       but optional is the only legacy one still in use for some lazy reason. */
 514    string OptionalStr = Tags.FindS("Optional");
 515    if (OptionalStr.empty() == false)
 516    {
 517       if (Tags.FindS("Suggests").empty() == false)
 518          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 519       Changes.push_back(pkgTagSection::Tag::Rewrite("Suggests", OptionalStr));
 520    }
 521
 522    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 523         I != OverItem->FieldOverride.end(); ++I)
 524       Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 525
 526    // Rewrite and store the fields.
 527    if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false ||
 528          Output->Write("\n", 1) == false)
 529       return false;
 530
 531    return Db.Finish();
 532 }
 533                                                                         /*}}}*/
 534 PackagesWriter::~PackagesWriter()                                       /*{{{*/
 535 {
 536 }
 537                                                                         /*}}}*/
 538
 539 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 540 // ---------------------------------------------------------------------
 541 /* Create a Translation-Master file for this Packages file */
 542 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 543                                         mode_t const &Permissions) : Comp(NULL), Output(NULL)
 544 {
 545    if (File.empty() == true)
 546       return;
 547
 548    Comp = new MultiCompress(File, TransCompress, Permissions);
 549    Output = &Comp->Input;
 550 }
 551                                                                         /*}}}*/
 552 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* Create a Translation-Master file for this Packages file */
 555 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 556                                   string const &MD5)
 557 {
 558    if (Output == NULL)
 559       return true;
 560
 561    // Different archs can include different versions and therefore
 562    // different descriptions - so we need to check for both name and md5.
 563    string const Record = Pkg + ":" + MD5;
 564
 565    if (Included.find(Record) != Included.end())
 566       return true;
 567
 568    std::string out;
 569    strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 570            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 571    Output->Write(out.c_str(), out.length());
 572
 573    Included.insert(Record);
 574    return true;
 575 }
 576                                                                         /*}}}*/
 577 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 578 // ---------------------------------------------------------------------
 579 /* */
 580 TranslationWriter::~TranslationWriter()
 581 {
 582    if (Comp != NULL)
 583       delete Comp;
 584 }
 585                                                                         /*}}}*/
 586
 587 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 588 // ---------------------------------------------------------------------
 589 /* */
 590 SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides,
 591                              string const &ExtOverrides) :
 592    FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats)
 593 {
 594    AddPattern("*.dsc");
 595    DeLinkLimit = 0;
 596    Buffer = 0;
 597    BufSize = 0;
 598
 599    // Process the command line options
 600    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Sources");
 601    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 602    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 603
 604    // Read the override file
 605    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 606       return;
 607    else
 608       NoOverride = true;
 609
 610    // WTF?? The logic above: if we can't read binary overrides, don't even try
 611    // reading source overrides. if we can read binary overrides, then say there
 612    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 613
 614    if (ExtOverrides.empty() == false)
 615       SOver.ReadExtraOverride(ExtOverrides);
 616
 617    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 618       SOver.ReadOverride(SOverrides,true);
 619 }
 620                                                                         /*}}}*/
 621 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 622 static std::string getDscHash(unsigned int const DoHashes,
 623       Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, char const * const FieldName,
 624       HashString const * const Hash, unsigned long long Size, std::string FileName)
 625 {
 626    if ((DoHashes & DoIt) != DoIt || Tags.Exists(FieldName) == false || Hash == NULL)
 627       return "";
 628    std::ostringstream out;
 629    out << "\n " << Hash->HashValue() << " " << Size << " " << FileName
 630       << "\n " << Tags.FindS(FieldName);
 631    return out.str();
 632 }
 633 bool SourcesWriter::DoPackage(string FileName)
 634 {
 635    // Pull all the data we need form the DB
 636    if (Db.GetFileInfo(FileName,
 637             false, /* DoControl */
 638             false, /* DoContents */
 639             false, /* GenContentsOnly */
 640             true, /* DoSource */
 641             DoHashes, DoAlwaysStat) == false)
 642    {
 643       return false;
 644    }
 645
 646    // we need to perform a "write" here (this is what finish is doing)
 647    // because the call to Db.GetFileInfo() in the loop will change
 648    // the "db cursor"
 649    Db.Finish();
 650
 651    pkgTagSection Tags;
 652    if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
 653       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 654
 655    if (Tags.Exists("Source") == false)
 656       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
 657    Tags.Trim();
 658
 659    // Lookup the overide information, finding first the best priority.
 660    string BestPrio;
 661    string Bins = Tags.FindS("Binary");
 662    char Buffer[Bins.length() + 1];
 663    unique_ptr<Override::Item> OverItem(nullptr);
 664    if (Bins.empty() == false)
 665    {
 666       strcpy(Buffer,Bins.c_str());
 667
 668       // Ignore too-long errors.
 669       char *BinList[400];
 670       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 671
 672       // Look at all the binaries
 673       unsigned char BestPrioV = pkgCache::State::Extra;
 674       for (unsigned I = 0; BinList[I] != 0; I++)
 675       {
 676          unique_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 677          if (Itm.get() == 0)
 678             continue;
 679
 680          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 681          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 682          {
 683             BestPrioV = NewPrioV;
 684             BestPrio = Itm->Priority;
 685          }
 686
 687          if (OverItem.get() == 0)
 688             OverItem = std::move(Itm);
 689       }
 690    }
 691
 692    // If we need to do any rewriting of the header do it now..
 693    if (OverItem.get() == 0)
 694    {
 695       if (NoOverride == false)
 696       {
 697          NewLine(1);
 698          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 699       }
 700
 701       OverItem.reset(new Override::Item);
 702    }
 703
 704    struct stat St;
 705    if (stat(FileName.c_str(), &St) != 0)
 706       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 707
 708    unique_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 709    // const unique_ptr<Override::Item> autoSOverItem(SOverItem);
 710    if (SOverItem.get() == 0)
 711    {
 712       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 713       SOverItem = unique_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 714       if (SOverItem.get() == 0)
 715       {
 716         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 717          SOverItem = unique_ptr<Override::Item>(new Override::Item);
 718          *SOverItem = *OverItem;
 719       }
 720    }
 721
 722    // Add the dsc to the files hash list
 723    string const strippedName = flNotDir(FileName);
 724    std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
 725    std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
 726    std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
 727    std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
 728
 729    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 730    string NewFileName;
 731    if (DirStrip.empty() == false &&
 732        FileName.length() > DirStrip.length() &&
 733        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 734       NewFileName = string(OriginalPath + DirStrip.length());
 735    else
 736       NewFileName = OriginalPath;
 737    if (PathPrefix.empty() == false)
 738       NewFileName = flCombine(PathPrefix,NewFileName);
 739
 740    string Directory = flNotFile(OriginalPath);
 741    string Package = Tags.FindS("Source");
 742
 743    // Perform operation over all of the files
 744    string ParseJnk;
 745    const char *C = Files.c_str();
 746    char *RealPath = NULL;
 747    for (;isspace(*C); C++);
 748    while (*C != 0)
 749    {
 750       // Parse each of the elements
 751       if (ParseQuoteWord(C,ParseJnk) == false ||
 752           ParseQuoteWord(C,ParseJnk) == false ||
 753           ParseQuoteWord(C,ParseJnk) == false)
 754          return _error->Error("Error parsing file record");
 755
 756       string OriginalPath = Directory + ParseJnk;
 757
 758       // Add missing hashes to source files
 759       if (((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM && !Tags.Exists("Checksums-Sha1")) ||
 760           ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM && !Tags.Exists("Checksums-Sha256")) ||
 761           ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM && !Tags.Exists("Checksums-Sha512")))
 762       {
 763          if (Db.GetFileInfo(OriginalPath,
 764                             false, /* DoControl */
 765                             false, /* DoContents */
 766                             false, /* GenContentsOnly */
 767                             false, /* DoSource */
 768                             DoHashes,
 769                             DoAlwaysStat) == false)
 770          {
 771             return _error->Error("Error getting file info");
 772          }
 773
 774          for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 775          {
 776             if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
 777                continue;
 778             char const * fieldname;
 779             std::string * out;
 780             if (hs->HashType() == "SHA1")
 781             {
 782                fieldname = "Checksums-Sha1";
 783                out = &ChecksumsSha1;
 784             }
 785             else if (hs->HashType() == "SHA256")
 786             {
 787                fieldname = "Checksums-Sha256";
 788                out = &ChecksumsSha256;
 789             }
 790             else if (hs->HashType() == "SHA512")
 791             {
 792                fieldname = "Checksums-Sha512";
 793                out = &ChecksumsSha512;
 794             }
 795             else
 796             {
 797                _error->Warning("Ignoring unknown Checksumtype %s in SourcesWriter::DoPackages", hs->HashType().c_str());
 798                continue;
 799             }
 800             if (Tags.Exists(fieldname) == true)
 801                continue;
 802             std::ostringstream streamout;
 803             streamout << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
 804             out->append(streamout.str());
 805          }
 806
 807          // write back the GetFileInfo() stats data
 808          Db.Finish();
 809       }
 810
 811       // Perform the delinking operation
 812       char Jnk[2];
 813
 814       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 815           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 816       {
 817          string RP = RealPath;
 818          free(RealPath);
 819          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 820             return false;
 821       }
 822    }
 823
 824    Directory = flNotFile(NewFileName);
 825    if (Directory.length() > 2)
 826       Directory.erase(Directory.end()-1);
 827
 828    // This lists all the changes to the fields we are going to make.
 829    // (5 hardcoded + checksums + maintainer + end marker)
 830    std::vector<pkgTagSection::Tag> Changes;
 831
 832    Changes.push_back(pkgTagSection::Tag::Remove("Source"));
 833    Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package));
 834    if (Files.empty() == false)
 835       Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files));
 836    if (ChecksumsSha1.empty() == false)
 837       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1));
 838    if (ChecksumsSha256.empty() == false)
 839       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256));
 840    if (ChecksumsSha512.empty() == false)
 841       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512));
 842    if (Directory != "./")
 843       Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory));
 844    Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio));
 845    Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 846
 847    // Rewrite the maintainer field if necessary
 848    bool MaintFailed;
 849    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"), MaintFailed);
 850    if (MaintFailed == true)
 851    {
 852       if (NoOverride == false)
 853       {
 854          NewLine(1);
 855          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 856                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 857       }
 858    }
 859    if (NewMaint.empty() == false)
 860       Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str()));
 861
 862    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 863         I != SOverItem->FieldOverride.end(); ++I)
 864       Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 865
 866    // Rewrite and store the fields.
 867    if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false ||
 868          Output->Write("\n", 1) == false)
 869       return false;
 870
 871    Stats.Packages++;
 872
 873    return true;
 874 }
 875                                                                         /*}}}*/
 876
 877 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 878 // ---------------------------------------------------------------------
 879 /* */
 880 ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB, string const &Arch) :
 881                     FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats)
 882
 883 {
 884    SetExts(".deb");
 885 }
 886                                                                         /*}}}*/
 887 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 888 // ---------------------------------------------------------------------
 889 /* If Package is the empty string the control record will be parsed to
 890    determine what the package name is. */
 891 bool ContentsWriter::DoPackage(string FileName, string Package)
 892 {
 893    if (!Db.GetFileInfo(FileName,
 894             Package.empty(), /* DoControl */
 895             true, /* DoContents */
 896             false, /* GenContentsOnly */
 897             false, /* DoSource */
 898             0, /* DoHashes */
 899             false /* checkMtime */))
 900    {
 901       return false;
 902    }
 903
 904    // Parse the package name
 905    if (Package.empty() == true)
 906    {
 907       Package = Db.Control.Section.FindS("Package");
 908    }
 909
 910    Db.Contents.Add(Gen,Package);
 911
 912    return Db.Finish();
 913 }
 914                                                                         /*}}}*/
 915 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* */
 918 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 919 {
 920    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 921    if (_error->PendingError() == true)
 922       return false;
 923
 924    // Open the package file
 925    FileFd Fd;
 926    if (Pkgs.OpenOld(Fd) == false)
 927       return false;
 928
 929    pkgTagFile Tags(&Fd);
 930    if (_error->PendingError() == true)
 931       return false;
 932
 933    // Parse.
 934    pkgTagSection Section;
 935    while (Tags.Step(Section) == true)
 936    {
 937       string File = flCombine(Prefix,Section.FindS("FileName"));
 938       string Package = Section.FindS("Section");
 939       if (Package.empty() == false && Package.end()[-1] != '/')
 940       {
 941          Package += '/';
 942          Package += Section.FindS("Package");
 943       }
 944       else
 945          Package += Section.FindS("Package");
 946
 947       DoPackage(File,Package);
 948       if (_error->empty() == false)
 949       {
 950          _error->Error("Errors apply to file '%s'",File.c_str());
 951          _error->DumpErrors();
 952       }
 953    }
 954
 955    // Tidy the compressor
 956    Fd.Close();
 957
 958    return true;
 959 }
 960
 961                                                                         /*}}}*/
 962
 963 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 964 // ---------------------------------------------------------------------
 965 /* */
 966 ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput)
 967 {
 968    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 969    {
 970       AddPattern("Packages");
 971       AddPattern("Packages.gz");
 972       AddPattern("Packages.bz2");
 973       AddPattern("Packages.lzma");
 974       AddPattern("Packages.xz");
 975       AddPattern("Translation-*");
 976       AddPattern("Sources");
 977       AddPattern("Sources.gz");
 978       AddPattern("Sources.bz2");
 979       AddPattern("Sources.lzma");
 980       AddPattern("Sources.xz");
 981       AddPattern("Release");
 982       AddPattern("Contents-*");
 983       AddPattern("Index");
 984       AddPattern("md5sum.txt");
 985    }
 986    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 987
 988    time_t const now = time(NULL);
 989
 990    setlocale(LC_TIME, "C");
 991
 992    char datestr[128];
 993    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 994                 gmtime(&now)) == 0)
 995    {
 996       datestr[0] = '\0';
 997    }
 998
 999    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
1000    char validstr[128];
1001    if (now == validuntil ||
1002        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
1003                 gmtime(&validuntil)) == 0)
1004    {
1005       validstr[0] = '\0';
1006    }
1007
1008    setlocale(LC_TIME, "");
1009
1010    map<string,string> Fields;
1011    Fields["Origin"] = "";
1012    Fields["Label"] = "";
1013    Fields["Suite"] = "";
1014    Fields["Version"] = "";
1015    Fields["Codename"] = "";
1016    Fields["Date"] = datestr;
1017    Fields["Valid-Until"] = validstr;
1018    Fields["Architectures"] = "";
1019    Fields["Components"] = "";
1020    Fields["Description"] = "";
1021
1022    for(map<string,string>::const_iterator I = Fields.begin();
1023        I != Fields.end();
1024        ++I)
1025    {
1026       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1027       string Value = _config->Find(Config, (*I).second.c_str());
1028       if (Value == "")
1029          continue;
1030
1031       std::string const out = I->first + ": " + Value + "\n";
1032       Output->Write(out.c_str(), out.length());
1033    }
1034
1035    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
1036 }
1037                                                                         /*}}}*/
1038 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1039 // ---------------------------------------------------------------------
1040 bool ReleaseWriter::DoPackage(string FileName)
1041 {
1042    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1043    string NewFileName;
1044    if (DirStrip.empty() == false &&
1045        FileName.length() > DirStrip.length() &&
1046        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1047                  DirStrip.begin(),DirStrip.end()) == 0)
1048    {
1049       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1050       while (NewFileName[0] == '/')
1051          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1052    }
1053    else
1054       NewFileName = FileName;
1055
1056    if (PathPrefix.empty() == false)
1057       NewFileName = flCombine(PathPrefix,NewFileName);
1058
1059    FileFd fd(FileName, FileFd::ReadOnly);
1060
1061    if (!fd.IsOpen())
1062    {
1063       return false;
1064    }
1065
1066    CheckSums[NewFileName].size = fd.Size();
1067
1068    Hashes hs(DoHashes);
1069    hs.AddFD(fd);
1070    CheckSums[NewFileName].Hashes = hs.GetHashStringList();
1071    fd.Close();
1072
1073    return true;
1074 }
1075
1076                                                                         /*}}}*/
1077 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1078 // ---------------------------------------------------------------------
1079 static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map<string, ReleaseWriter::CheckSum> const &CheckSums)
1080 {
1081    {
1082       std::string out;
1083       strprintf(out, "%s:\n", Type);
1084       Output.Write(out.c_str(), out.length());
1085    }
1086    for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
1087          I != CheckSums.end(); ++I)
1088    {
1089       HashString const * const hs = I->second.Hashes.find(Type);
1090       if (hs == NULL)
1091          continue;
1092       std::string out;
1093       strprintf(out, " %s %16llu %s\n",
1094             hs->HashValue().c_str(),
1095             (*I).second.size,
1096             (*I).first.c_str());
1097       Output.Write(out.c_str(), out.length());
1098    }
1099 }
1100 void ReleaseWriter::Finish()
1101 {
1102    if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
1103       printChecksumTypeRecord(*Output, "MD5Sum", CheckSums);
1104    if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
1105       printChecksumTypeRecord(*Output, "SHA1", CheckSums);
1106    if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
1107       printChecksumTypeRecord(*Output, "SHA256", CheckSums);
1108    if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
1109       printChecksumTypeRecord(*Output, "SHA512", CheckSums);
1110 }