ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/deblistparser.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/fileutl.h>
  20 #include <apt-pkg/gpgv.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/debfile.h>
  25 #include <apt-pkg/pkgcache.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/sha2.h>
  28 #include <apt-pkg/tagfile.h>
  29
  30 #include <ctype.h>
  31 #include <fnmatch.h>
  32 #include <ftw.h>
  33 #include <locale.h>
  34 #include <string.h>
  35 #include <sys/stat.h>
  36 #include <sys/types.h>
  37 #include <unistd.h>
  38 #include <ctime>
  39 #include <iostream>
  40 #include <sstream>
  41 #include <memory>
  42 #include <utility>
  43 #include <algorithm>
  44
  45 #include "apt-ftparchive.h"
  46 #include "writer.h"
  47 #include "cachedb.h"
  48 #include "multicompress.h"
  49 #include "byhash.h"
  50
  51 #include <apti18n.h>
  52                                                                         /*}}}*/
  53 using namespace std;
  54 FTWScanner *FTWScanner::Owner;
  55
  56 // ConfigToDoHashes - which hashes to generate                          /*{{{*/
  57 static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
  58 {
  59    if (_config->FindB(Conf, (DoHashes & Flag) == Flag) == true)
  60       DoHashes |= Flag;
  61    else
  62       DoHashes &= ~Flag;
  63 }
  64 static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
  65 {
  66    SingleConfigToDoHashes(DoHashes, Conf + "::MD5", Hashes::MD5SUM);
  67    SingleConfigToDoHashes(DoHashes, Conf + "::SHA1", Hashes::SHA1SUM);
  68    SingleConfigToDoHashes(DoHashes, Conf + "::SHA256", Hashes::SHA256SUM);
  69    SingleConfigToDoHashes(DoHashes, Conf + "::SHA512", Hashes::SHA512SUM);
  70 }
  71                                                                         /*}}}*/
  72
  73 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  74 FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch, bool const IncludeArchAll)
  75    : Arch(Arch), IncludeArchAll(IncludeArchAll), DoHashes(~0)
  76 {
  77    if (GivenOutput == NULL)
  78    {
  79       Output = new FileFd;
  80       OwnsOutput = true;
  81       Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
  82    }
  83    else
  84    {
  85       Output = GivenOutput;
  86       OwnsOutput = false;
  87    }
  88    ErrorPrinted = false;
  89    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  90    ConfigToDoHashes(DoHashes, "APT::FTPArchive");
  91 }
  92                                                                         /*}}}*/
  93 FTWScanner::~FTWScanner()
  94 {
  95    if (Output != NULL && OwnsOutput)
  96       delete Output;
  97 }
  98 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* This is the FTW scanner, it processes each directory element in the
 101    directory tree. */
 102 int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
 103 {
 104    if (Flag == FTW_DNR)
 105    {
 106       Owner->NewLine(1);
 107       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
 108    }
 109    if (Flag == FTW_NS)
 110    {
 111       Owner->NewLine(1);
 112       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
 113    }
 114    if (Flag != FTW_F)
 115       return 0;
 116
 117    return ScannerFile(File, true);
 118 }
 119                                                                         /*}}}*/
 120 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
 121 // ---------------------------------------------------------------------
 122 /* */
 123 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
 124 {
 125    const char *LastComponent = strrchr(File, '/');
 126    char *RealPath = NULL;
 127
 128    if (LastComponent == NULL)
 129       LastComponent = File;
 130    else
 131       LastComponent++;
 132
 133    vector<string>::const_iterator I;
 134    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 135    {
 136       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 137          break;
 138    }
 139    if (I == Owner->Patterns.end())
 140       return 0;
 141
 142    /* Process it. If the file is a link then resolve it into an absolute
 143       name.. This works best if the directory components the scanner are
 144       given are not links themselves. */
 145    char Jnk[2];
 146    Owner->OriginalPath = File;
 147    if (ReadLink &&
 148        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 149        (RealPath = realpath(File,NULL)) != 0)
 150    {
 151       Owner->DoPackage(RealPath);
 152       free(RealPath);
 153    }
 154    else
 155       Owner->DoPackage(File);
 156
 157    if (_error->empty() == false)
 158    {
 159       // Print any errors or warnings found
 160       string Err;
 161       bool SeenPath = false;
 162       while (_error->empty() == false)
 163       {
 164          Owner->NewLine(1);
 165
 166          bool const Type = _error->PopMessage(Err);
 167          if (Type == true)
 168             cerr << _("E: ") << Err << endl;
 169          else
 170             cerr << _("W: ") << Err << endl;
 171
 172          if (Err.find(File) != string::npos)
 173             SeenPath = true;
 174       }
 175
 176       if (SeenPath == false)
 177          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 178       return 0;
 179    }
 180
 181    return 0;
 182 }
 183                                                                         /*}}}*/
 184 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 185 // ---------------------------------------------------------------------
 186 /* */
 187 bool FTWScanner::RecursiveScan(string const &Dir)
 188 {
 189    char *RealPath = NULL;
 190    /* If noprefix is set then jam the scan root in, so we don't generate
 191       link followed paths out of control */
 192    if (InternalPrefix.empty() == true)
 193    {
 194       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 195          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 196       InternalPrefix = RealPath;
 197       free(RealPath);
 198    }
 199
 200    // Do recursive directory searching
 201    Owner = this;
 202    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 203
 204    // Error treewalking?
 205    if (Res != 0)
 206    {
 207       if (_error->PendingError() == false)
 208          _error->Errno("ftw",_("Tree walking failed"));
 209       return false;
 210    }
 211
 212    return true;
 213 }
 214                                                                         /*}}}*/
 215 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 216 // ---------------------------------------------------------------------
 217 /* This is an alternative to using FTW to locate files, it reads the list
 218    of files from another file. */
 219 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 220 {
 221    char *RealPath = NULL;
 222    /* If noprefix is set then jam the scan root in, so we don't generate
 223       link followed paths out of control */
 224    if (InternalPrefix.empty() == true)
 225    {
 226       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 227          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 228       InternalPrefix = RealPath;
 229       free(RealPath);
 230    }
 231
 232    Owner = this;
 233    FILE *List = fopen(File.c_str(),"r");
 234    if (List == 0)
 235       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 236
 237    /* We are a tad tricky here.. We prefix the buffer with the directory
 238       name, that way if we need a full path with just use line.. Sneaky and
 239       fully evil. */
 240    char Line[1000];
 241    char *FileStart;
 242    if (Dir.empty() == true || Dir.end()[-1] != '/')
 243       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 244    else
 245       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 246    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 247    {
 248       char *FileName = _strstrip(FileStart);
 249       if (FileName[0] == 0)
 250          continue;
 251
 252       if (FileName[0] != '/')
 253       {
 254          if (FileName != FileStart)
 255             memmove(FileStart,FileName,strlen(FileStart));
 256          FileName = Line;
 257       }
 258
 259 #if 0
 260       struct stat St;
 261       int Flag = FTW_F;
 262       if (stat(FileName,&St) != 0)
 263          Flag = FTW_NS;
 264 #endif
 265
 266       if (ScannerFile(FileName, false) != 0)
 267          break;
 268    }
 269
 270    fclose(List);
 271    return true;
 272 }
 273                                                                         /*}}}*/
 274 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 275 // ---------------------------------------------------------------------
 276 /* */
 277 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 278                         unsigned long long &DeLinkBytes,
 279                         unsigned long long const &FileSize)
 280 {
 281    // See if this isn't an internaly prefix'd file name.
 282    if (InternalPrefix.empty() == false &&
 283        InternalPrefix.length() < FileName.length() &&
 284        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 285                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 286    {
 287       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 288       {
 289          // Tidy up the display
 290          if (DeLinkBytes == 0)
 291             cout << endl;
 292
 293          NewLine(1);
 294          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 295                     SizeToStr(FileSize).c_str());
 296          c1out << flush;
 297
 298          if (NoLinkAct == false)
 299          {
 300             char OldLink[400];
 301             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 302                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 303             else
 304             {
 305                if (RemoveFile("FTWScanner::Delink", OriginalPath))
 306                {
 307                   if (link(FileName.c_str(),OriginalPath) != 0)
 308                   {
 309                      // Panic! Restore the symlink
 310                      if (symlink(OldLink,OriginalPath) != 0)
 311                         _error->Errno("symlink", "failed to restore symlink");
 312                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 313                                           FileName.c_str(),
 314                                           OriginalPath);
 315                   }
 316                }
 317             }
 318          }
 319
 320          DeLinkBytes += FileSize;
 321          if (DeLinkBytes/1024 >= DeLinkLimit)
 322             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 323       }
 324
 325       FileName = OriginalPath;
 326    }
 327
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* */
 334 bool FTWScanner::SetExts(string const &Vals)
 335 {
 336    ClearPatterns();
 337    string::size_type Start = 0;
 338    while (Start <= Vals.length()-1)
 339    {
 340       string::size_type const Space = Vals.find(' ',Start);
 341       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 342       if ( Arch.empty() == false )
 343       {
 344          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 345          if (IncludeArchAll == true && Arch != "all")
 346             AddPattern(string("*_all") + Vals.substr(Start, Length));
 347       }
 348       else
 349          AddPattern(string("*") + Vals.substr(Start, Length));
 350
 351       Start += Length + 1;
 352    }
 353
 354    return true;
 355 }
 356                                                                         /*}}}*/
 357
 358 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 359 // ---------------------------------------------------------------------
 360 /* */
 361 PackagesWriter::PackagesWriter(FileFd * const GivenOutput, TranslationWriter * const transWriter,
 362       string const &DB,string const &Overrides,string const &ExtOverrides,
 363       string const &Arch, bool const IncludeArchAll) :
 364    FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats), TransWriter(transWriter)
 365 {
 366    SetExts(".deb .udeb");
 367    DeLinkLimit = 0;
 368
 369    // Process the command line options
 370    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Packages");
 371    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 372    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 373    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 374    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 375
 376    if (Db.Loaded() == false)
 377       DoContents = false;
 378
 379    // Read the override file
 380    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 381       return;
 382    else
 383       NoOverride = true;
 384
 385    if (ExtOverrides.empty() == false)
 386       Over.ReadExtraOverride(ExtOverrides);
 387
 388    _error->DumpErrors();
 389 }
 390                                                                         /*}}}*/
 391 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 392 // ---------------------------------------------------------------------
 393 /* This method takes a package and gets its control information and
 394    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 395    rewritten and the path/size/hash appended. */
 396 bool PackagesWriter::DoPackage(string FileName)
 397 {
 398    // Pull all the data we need form the DB
 399    if (Db.GetFileInfo(FileName,
 400             true, /* DoControl */
 401             DoContents,
 402             true, /* GenContentsOnly */
 403             false, /* DoSource */
 404             DoHashes, DoAlwaysStat) == false)
 405    {
 406      return false;
 407    }
 408
 409    unsigned long long FileSize = Db.GetFileSize();
 410    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 411       return false;
 412
 413    // Lookup the overide information
 414    pkgTagSection &Tags = Db.Control.Section;
 415    string Package = Tags.FindS("Package");
 416    string Architecture;
 417    // if we generate a Packages file for a given arch, we use it to
 418    // look for overrides. if we run in "simple" mode without the
 419    // "Architecures" variable in the config we use the architecure value
 420    // from the deb file
 421    if(Arch != "")
 422       Architecture = Arch;
 423    else
 424       Architecture = Tags.FindS("Architecture");
 425    unique_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 426
 427    if (Package.empty() == true)
 428       return _error->Error(_("Archive had no package field"));
 429
 430    // If we need to do any rewriting of the header do it now..
 431    if (OverItem.get() == 0)
 432    {
 433       if (NoOverride == false)
 434       {
 435          NewLine(1);
 436          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 437       }
 438
 439       OverItem = unique_ptr<Override::Item>(new Override::Item);
 440       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 441       OverItem->Priority = Tags.FindS("Priority");
 442    }
 443
 444    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 445    string NewFileName;
 446    if (DirStrip.empty() == false &&
 447        FileName.length() > DirStrip.length() &&
 448        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 449                  DirStrip.begin(),DirStrip.end()) == 0)
 450       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 451    else
 452       NewFileName = FileName;
 453    if (PathPrefix.empty() == false)
 454       NewFileName = flCombine(PathPrefix,NewFileName);
 455
 456    /* Configuration says we don't want to include the long Description
 457       in the package file - instead we want to ship a separated file */
 458    string desc;
 459    if (LongDescription == false) {
 460       desc = Tags.FindS("Description").append("\n");
 461       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 462    }
 463
 464    // This lists all the changes to the fields we are going to make.
 465    std::vector<pkgTagSection::Tag> Changes;
 466
 467    std::string Size;
 468    strprintf(Size, "%llu", (unsigned long long) FileSize);
 469    Changes.push_back(pkgTagSection::Tag::Rewrite("Size", Size));
 470
 471    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 472    {
 473       if (hs->HashType() == "MD5Sum")
 474          Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue()));
 475       else if (hs->HashType() == "Checksum-FileSize")
 476          continue;
 477       else
 478          Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue()));
 479    }
 480    Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName));
 481    Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority));
 482    Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 483    Changes.push_back(pkgTagSection::Tag::Remove("Optional"));
 484
 485    string DescriptionMd5;
 486    if (LongDescription == false) {
 487       MD5Summation descmd5;
 488       descmd5.Add(desc.c_str());
 489       DescriptionMd5 = descmd5.Result().Value();
 490       Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5));
 491       if (TransWriter != NULL)
 492          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 493    }
 494
 495    // Rewrite the maintainer field if necessary
 496    bool MaintFailed;
 497    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 498    if (MaintFailed == true)
 499    {
 500       if (NoOverride == false)
 501       {
 502          NewLine(1);
 503          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 504                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 505       }
 506    }
 507
 508    if (NewMaint.empty() == false)
 509       Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint));
 510
 511    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 512       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 513       but dpkg does this append bit. So we do the append bit, at least that way the
 514       status file and package file will remain similar. There are other transforms
 515       but optional is the only legacy one still in use for some lazy reason. */
 516    string OptionalStr = Tags.FindS("Optional");
 517    if (OptionalStr.empty() == false)
 518    {
 519       if (Tags.FindS("Suggests").empty() == false)
 520          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 521       Changes.push_back(pkgTagSection::Tag::Rewrite("Suggests", OptionalStr));
 522    }
 523
 524    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 525         I != OverItem->FieldOverride.end(); ++I)
 526       Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 527
 528    // Rewrite and store the fields.
 529    if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false ||
 530          Output->Write("\n", 1) == false)
 531       return false;
 532
 533    return Db.Finish();
 534 }
 535                                                                         /*}}}*/
 536 PackagesWriter::~PackagesWriter()                                       /*{{{*/
 537 {
 538 }
 539                                                                         /*}}}*/
 540
 541 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 542 // ---------------------------------------------------------------------
 543 /* Create a Translation-Master file for this Packages file */
 544 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 545                                         mode_t const &Permissions) : Comp(NULL), Output(NULL)
 546 {
 547    if (File.empty() == true)
 548       return;
 549
 550    Comp = new MultiCompress(File, TransCompress, Permissions);
 551    Output = &Comp->Input;
 552 }
 553                                                                         /*}}}*/
 554 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 555 // ---------------------------------------------------------------------
 556 /* Create a Translation-Master file for this Packages file */
 557 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 558                                   string const &MD5)
 559 {
 560    if (Output == NULL)
 561       return true;
 562
 563    // Different archs can include different versions and therefore
 564    // different descriptions - so we need to check for both name and md5.
 565    string const Record = Pkg + ":" + MD5;
 566
 567    if (Included.find(Record) != Included.end())
 568       return true;
 569
 570    std::string out;
 571    strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 572            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 573    Output->Write(out.c_str(), out.length());
 574
 575    Included.insert(Record);
 576    return true;
 577 }
 578                                                                         /*}}}*/
 579 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 580 // ---------------------------------------------------------------------
 581 /* */
 582 TranslationWriter::~TranslationWriter()
 583 {
 584    if (Comp != NULL)
 585       delete Comp;
 586 }
 587                                                                         /*}}}*/
 588
 589 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 590 // ---------------------------------------------------------------------
 591 /* */
 592 SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides,
 593                              string const &ExtOverrides) :
 594    FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats)
 595 {
 596    AddPattern("*.dsc");
 597    DeLinkLimit = 0;
 598    Buffer = 0;
 599    BufSize = 0;
 600
 601    // Process the command line options
 602    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Sources");
 603    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 604    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 605
 606    // Read the override file
 607    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 608       return;
 609    else
 610       NoOverride = true;
 611
 612    // WTF?? The logic above: if we can't read binary overrides, don't even try
 613    // reading source overrides. if we can read binary overrides, then say there
 614    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 615
 616    if (ExtOverrides.empty() == false)
 617       SOver.ReadExtraOverride(ExtOverrides);
 618
 619    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 620       SOver.ReadOverride(SOverrides,true);
 621 }
 622                                                                         /*}}}*/
 623 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 624 static std::string getDscHash(unsigned int const DoHashes,
 625       Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, char const * const FieldName,
 626       HashString const * const Hash, unsigned long long Size, std::string FileName)
 627 {
 628    if ((DoHashes & DoIt) != DoIt || Tags.Exists(FieldName) == false || Hash == NULL)
 629       return "";
 630    std::ostringstream out;
 631    out << "\n " << Hash->HashValue() << " " << Size << " " << FileName
 632       << "\n " << Tags.FindS(FieldName);
 633    return out.str();
 634 }
 635 bool SourcesWriter::DoPackage(string FileName)
 636 {
 637    // Pull all the data we need form the DB
 638    if (Db.GetFileInfo(FileName,
 639             false, /* DoControl */
 640             false, /* DoContents */
 641             false, /* GenContentsOnly */
 642             true, /* DoSource */
 643             DoHashes, DoAlwaysStat) == false)
 644    {
 645       return false;
 646    }
 647
 648    // we need to perform a "write" here (this is what finish is doing)
 649    // because the call to Db.GetFileInfo() in the loop will change
 650    // the "db cursor"
 651    Db.Finish();
 652
 653    pkgTagSection Tags;
 654    if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
 655       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 656
 657    if (Tags.Exists("Source") == false)
 658       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
 659    Tags.Trim();
 660
 661    // Lookup the overide information, finding first the best priority.
 662    string BestPrio;
 663    string Bins = Tags.FindS("Binary");
 664    char Buffer[Bins.length() + 1];
 665    unique_ptr<Override::Item> OverItem(nullptr);
 666    if (Bins.empty() == false)
 667    {
 668       strcpy(Buffer,Bins.c_str());
 669
 670       // Ignore too-long errors.
 671       char *BinList[400];
 672       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 673
 674       // Look at all the binaries
 675       unsigned char BestPrioV = pkgCache::State::Extra;
 676       for (unsigned I = 0; BinList[I] != 0; I++)
 677       {
 678          unique_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 679          if (Itm.get() == 0)
 680             continue;
 681
 682          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 683          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 684          {
 685             BestPrioV = NewPrioV;
 686             BestPrio = Itm->Priority;
 687          }
 688
 689          if (OverItem.get() == 0)
 690             OverItem = std::move(Itm);
 691       }
 692    }
 693
 694    // If we need to do any rewriting of the header do it now..
 695    if (OverItem.get() == 0)
 696    {
 697       if (NoOverride == false)
 698       {
 699          NewLine(1);
 700          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 701       }
 702
 703       OverItem.reset(new Override::Item);
 704    }
 705
 706    struct stat St;
 707    if (stat(FileName.c_str(), &St) != 0)
 708       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 709
 710    unique_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 711    // const unique_ptr<Override::Item> autoSOverItem(SOverItem);
 712    if (SOverItem.get() == 0)
 713    {
 714       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 715       SOverItem = unique_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 716       if (SOverItem.get() == 0)
 717       {
 718         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 719          SOverItem = unique_ptr<Override::Item>(new Override::Item);
 720          *SOverItem = *OverItem;
 721       }
 722    }
 723
 724    // Add the dsc to the files hash list
 725    string const strippedName = flNotDir(FileName);
 726    std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
 727    std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
 728    std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
 729    std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
 730
 731    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 732    string NewFileName;
 733    if (DirStrip.empty() == false &&
 734        FileName.length() > DirStrip.length() &&
 735        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 736       NewFileName = string(OriginalPath + DirStrip.length());
 737    else
 738       NewFileName = OriginalPath;
 739    if (PathPrefix.empty() == false)
 740       NewFileName = flCombine(PathPrefix,NewFileName);
 741
 742    string Directory = flNotFile(OriginalPath);
 743    string Package = Tags.FindS("Source");
 744
 745    // Perform operation over all of the files
 746    string ParseJnk;
 747    const char *C = Files.c_str();
 748    char *RealPath = NULL;
 749    for (;isspace(*C); C++);
 750    while (*C != 0)
 751    {
 752       // Parse each of the elements
 753       if (ParseQuoteWord(C,ParseJnk) == false ||
 754           ParseQuoteWord(C,ParseJnk) == false ||
 755           ParseQuoteWord(C,ParseJnk) == false)
 756          return _error->Error("Error parsing file record");
 757
 758       string OriginalPath = Directory + ParseJnk;
 759
 760       // Add missing hashes to source files
 761       if (((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM && !Tags.Exists("Checksums-Sha1")) ||
 762           ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM && !Tags.Exists("Checksums-Sha256")) ||
 763           ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM && !Tags.Exists("Checksums-Sha512")))
 764       {
 765          if (Db.GetFileInfo(OriginalPath,
 766                             false, /* DoControl */
 767                             false, /* DoContents */
 768                             false, /* GenContentsOnly */
 769                             false, /* DoSource */
 770                             DoHashes,
 771                             DoAlwaysStat) == false)
 772          {
 773             return _error->Error("Error getting file info");
 774          }
 775
 776          for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 777          {
 778             if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
 779                continue;
 780             char const * fieldname;
 781             std::string * out;
 782             if (hs->HashType() == "SHA1")
 783             {
 784                fieldname = "Checksums-Sha1";
 785                out = &ChecksumsSha1;
 786             }
 787             else if (hs->HashType() == "SHA256")
 788             {
 789                fieldname = "Checksums-Sha256";
 790                out = &ChecksumsSha256;
 791             }
 792             else if (hs->HashType() == "SHA512")
 793             {
 794                fieldname = "Checksums-Sha512";
 795                out = &ChecksumsSha512;
 796             }
 797             else
 798             {
 799                _error->Warning("Ignoring unknown Checksumtype %s in SourcesWriter::DoPackages", hs->HashType().c_str());
 800                continue;
 801             }
 802             if (Tags.Exists(fieldname) == true)
 803                continue;
 804             std::ostringstream streamout;
 805             streamout << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
 806             out->append(streamout.str());
 807          }
 808
 809          // write back the GetFileInfo() stats data
 810          Db.Finish();
 811       }
 812
 813       // Perform the delinking operation
 814       char Jnk[2];
 815
 816       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 817           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 818       {
 819          string RP = RealPath;
 820          free(RealPath);
 821          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 822             return false;
 823       }
 824    }
 825
 826    Directory = flNotFile(NewFileName);
 827    if (Directory.length() > 2)
 828       Directory.erase(Directory.end()-1);
 829
 830    // This lists all the changes to the fields we are going to make.
 831    // (5 hardcoded + checksums + maintainer + end marker)
 832    std::vector<pkgTagSection::Tag> Changes;
 833
 834    Changes.push_back(pkgTagSection::Tag::Remove("Source"));
 835    Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package));
 836    if (Files.empty() == false)
 837       Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files));
 838    if (ChecksumsSha1.empty() == false)
 839       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1));
 840    if (ChecksumsSha256.empty() == false)
 841       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256));
 842    if (ChecksumsSha512.empty() == false)
 843       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512));
 844    if (Directory != "./")
 845       Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory));
 846    Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio));
 847    Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 848
 849    // Rewrite the maintainer field if necessary
 850    bool MaintFailed;
 851    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"), MaintFailed);
 852    if (MaintFailed == true)
 853    {
 854       if (NoOverride == false)
 855       {
 856          NewLine(1);
 857          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 858                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 859       }
 860    }
 861    if (NewMaint.empty() == false)
 862       Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str()));
 863
 864    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 865         I != SOverItem->FieldOverride.end(); ++I)
 866       Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 867
 868    // Rewrite and store the fields.
 869    if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false ||
 870          Output->Write("\n", 1) == false)
 871       return false;
 872
 873    Stats.Packages++;
 874
 875    return true;
 876 }
 877                                                                         /*}}}*/
 878
 879 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 880 // ---------------------------------------------------------------------
 881 /* */
 882 ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB,
 883       string const &Arch, bool const IncludeArchAll) :
 884                     FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats)
 885
 886 {
 887    SetExts(".deb");
 888 }
 889                                                                         /*}}}*/
 890 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 891 // ---------------------------------------------------------------------
 892 /* If Package is the empty string the control record will be parsed to
 893    determine what the package name is. */
 894 bool ContentsWriter::DoPackage(string FileName, string Package)
 895 {
 896    if (!Db.GetFileInfo(FileName,
 897             Package.empty(), /* DoControl */
 898             true, /* DoContents */
 899             false, /* GenContentsOnly */
 900             false, /* DoSource */
 901             0, /* DoHashes */
 902             false /* checkMtime */))
 903    {
 904       return false;
 905    }
 906
 907    // Parse the package name
 908    if (Package.empty() == true)
 909    {
 910       Package = Db.Control.Section.FindS("Package");
 911    }
 912
 913    Db.Contents.Add(Gen,Package);
 914
 915    return Db.Finish();
 916 }
 917                                                                         /*}}}*/
 918 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 919 // ---------------------------------------------------------------------
 920 /* */
 921 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 922 {
 923    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 924    if (_error->PendingError() == true)
 925       return false;
 926
 927    // Open the package file
 928    FileFd Fd;
 929    if (Pkgs.OpenOld(Fd) == false)
 930       return false;
 931
 932    pkgTagFile Tags(&Fd);
 933    if (_error->PendingError() == true)
 934       return false;
 935
 936    // Parse.
 937    pkgTagSection Section;
 938    while (Tags.Step(Section) == true)
 939    {
 940       string File = flCombine(Prefix,Section.FindS("FileName"));
 941       string Package = Section.FindS("Section");
 942       if (Package.empty() == false && Package.end()[-1] != '/')
 943       {
 944          Package += '/';
 945          Package += Section.FindS("Package");
 946       }
 947       else
 948          Package += Section.FindS("Package");
 949
 950       DoPackage(File,Package);
 951       if (_error->empty() == false)
 952       {
 953          _error->Error("Errors apply to file '%s'",File.c_str());
 954          _error->DumpErrors();
 955       }
 956    }
 957
 958    // Tidy the compressor
 959    Fd.Close();
 960
 961    return true;
 962 }
 963
 964                                                                         /*}}}*/
 965
 966 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 967 // ---------------------------------------------------------------------
 968 /* */
 969 ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput)
 970 {
 971    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 972    {
 973       AddPattern("Packages");
 974       AddPattern("Packages.*");
 975       AddPattern("Translation-*");
 976       AddPattern("Sources");
 977       AddPattern("Sources.*");
 978       AddPattern("Release");
 979       AddPattern("Contents-*");
 980       AddPattern("Index");
 981       AddPattern("icons-*.tar");
 982       AddPattern("icons-*.tar.*");
 983       AddPattern("Components-*.yml");
 984       AddPattern("Components-*.yml.*");
 985       AddPattern("md5sum.txt");
 986    }
 987    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 988
 989    time_t const now = time(NULL);
 990
 991    setlocale(LC_TIME, "C");
 992
 993    char datestr[128];
 994    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 995                 gmtime(&now)) == 0)
 996    {
 997       datestr[0] = '\0';
 998    }
 999
1000    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
1001    char validstr[128];
1002    if (now == validuntil ||
1003        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
1004                 gmtime(&validuntil)) == 0)
1005    {
1006       validstr[0] = '\0';
1007    }
1008
1009    setlocale(LC_TIME, "");
1010
1011    map<string,string> Fields;
1012    Fields["Origin"] = "";
1013    Fields["Label"] = "";
1014    Fields["Suite"] = "";
1015    Fields["Version"] = "";
1016    Fields["Codename"] = "";
1017    Fields["Date"] = datestr;
1018    Fields["Valid-Until"] = validstr;
1019    Fields["Architectures"] = "";
1020    Fields["Components"] = "";
1021    Fields["Description"] = "";
1022    Fields["Signed-By"] = "";
1023    if (_config->FindB("APT::FTPArchive::DoByHash", false) == true)
1024       Fields["Acquire-By-Hash"] = "true";
1025
1026    for(map<string,string>::const_iterator I = Fields.begin();
1027        I != Fields.end();
1028        ++I)
1029    {
1030       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1031       string Value = _config->Find(Config, (*I).second.c_str());
1032       if (Value == "")
1033          continue;
1034
1035       std::string const out = I->first + ": " + Value + "\n";
1036       Output->Write(out.c_str(), out.length());
1037    }
1038
1039    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
1040 }
1041                                                                         /*}}}*/
1042 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1043 // ---------------------------------------------------------------------
1044 bool ReleaseWriter::DoPackage(string FileName)
1045 {
1046    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1047    string NewFileName;
1048    if (DirStrip.empty() == false &&
1049        FileName.length() > DirStrip.length() &&
1050        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1051                  DirStrip.begin(),DirStrip.end()) == 0)
1052    {
1053       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1054       while (NewFileName[0] == '/')
1055          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1056    }
1057    else
1058       NewFileName = FileName;
1059
1060    if (PathPrefix.empty() == false)
1061       NewFileName = flCombine(PathPrefix,NewFileName);
1062
1063    FileFd fd(FileName, FileFd::ReadOnly);
1064
1065    if (!fd.IsOpen())
1066    {
1067       return false;
1068    }
1069
1070    CheckSums[NewFileName].size = fd.Size();
1071
1072    Hashes hs(DoHashes);
1073    hs.AddFD(fd);
1074    CheckSums[NewFileName].Hashes = hs.GetHashStringList();
1075    fd.Close();
1076
1077    // FIXME: wrong layer in the code(?)
1078    // FIXME2: symlink instead of create a copy
1079    if (_config->FindB("APT::FTPArchive::DoByHash", false) == true)
1080    {
1081       std::string Input = FileName;
1082       HashStringList hsl = hs.GetHashStringList();
1083       for(HashStringList::const_iterator h = hsl.begin();
1084           h != hsl.end(); ++h)
1085       {
1086          if (!h->usable())
1087             continue;
1088          if (flNotDir(FileName) == "Release" || flNotDir(FileName) == "InRelease")
1089             continue;
1090
1091          std::string ByHashOutputFile = GenByHashFilename(Input, *h);
1092          std::string ByHashOutputDir = flNotFile(ByHashOutputFile);
1093          if(!CreateDirectory(flNotFile(Input), ByHashOutputDir))
1094             return _error->Warning("can not create dir %s", flNotFile(ByHashOutputFile).c_str());
1095
1096          // write new hashes
1097          FileFd In(Input, FileFd::ReadOnly);
1098          FileFd Out(ByHashOutputFile, FileFd::WriteEmpty);
1099          if(!CopyFile(In, Out))
1100             return _error->Warning("failed to copy %s %s", Input.c_str(), ByHashOutputFile.c_str());
1101       }
1102    }
1103
1104    return true;
1105 }
1106
1107                                                                         /*}}}*/
1108 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1109 // ---------------------------------------------------------------------
1110 static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map<string, ReleaseWriter::CheckSum> const &CheckSums)
1111 {
1112    {
1113       std::string out;
1114       strprintf(out, "%s:\n", Type);
1115       Output.Write(out.c_str(), out.length());
1116    }
1117    for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
1118          I != CheckSums.end(); ++I)
1119    {
1120       HashString const * const hs = I->second.Hashes.find(Type);
1121       if (hs == NULL)
1122          continue;
1123       std::string out;
1124       strprintf(out, " %s %16llu %s\n",
1125             hs->HashValue().c_str(),
1126             (*I).second.size,
1127             (*I).first.c_str());
1128       Output.Write(out.c_str(), out.length());
1129    }
1130 }
1131 void ReleaseWriter::Finish()
1132 {
1133    if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
1134       printChecksumTypeRecord(*Output, "MD5Sum", CheckSums);
1135    if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
1136       printChecksumTypeRecord(*Output, "SHA1", CheckSums);
1137    if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
1138       printChecksumTypeRecord(*Output, "SHA256", CheckSums);
1139    if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
1140       printChecksumTypeRecord(*Output, "SHA512", CheckSums);
1141
1142    // go by-hash cleanup
1143    map<string,ReleaseWriter::CheckSum>::const_iterator prev = CheckSums.begin();
1144    if (_config->FindB("APT::FTPArchive::DoByHash", false) == true)
1145    {
1146       for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
1147          I != CheckSums.end(); ++I)
1148       {
1149          if (I->first == "Release" || I->first == "InRelease")
1150             continue;
1151
1152          // keep iterating until we find a new subdir
1153          if(flNotFile(I->first) == flNotFile(prev->first))
1154             continue;
1155
1156          // clean that subdir up
1157          int keepFiles = _config->FindI("APT::FTPArchive::By-Hash-Keep", 3);
1158          // calculate how many compressors are used (the amount of files
1159          // in that subdir generated for this run)
1160          keepFiles *= std::distance(prev, I);
1161          prev = I;
1162
1163          HashStringList hsl = prev->second.Hashes;
1164          for(HashStringList::const_iterator h = hsl.begin();
1165              h != hsl.end(); ++h)
1166          {
1167
1168             if (!h->usable())
1169                continue;
1170
1171             std::string RealFilename = DirStrip+"/"+prev->first;
1172             std::string ByHashOutputFile = GenByHashFilename(RealFilename, *h);
1173             DeleteAllButMostRecent(flNotFile(ByHashOutputFile), keepFiles);
1174          }
1175       }
1176    }
1177 }