ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/deblistparser.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/fileutl.h>
  20 #include <apt-pkg/gpgv.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/debfile.h>
  25 #include <apt-pkg/pkgcache.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/sha2.h>
  28 #include <apt-pkg/tagfile.h>
  29
  30 #include <ctype.h>
  31 #include <fnmatch.h>
  32 #include <ftw.h>
  33 #include <locale.h>
  34 #include <string.h>
  35 #include <sys/stat.h>
  36 #include <sys/types.h>
  37 #include <unistd.h>
  38 #include <ctime>
  39 #include <iostream>
  40 #include <sstream>
  41 #include <memory>
  42 #include <utility>
  43
  44 #include "apt-ftparchive.h"
  45 #include "writer.h"
  46 #include "cachedb.h"
  47 #include "multicompress.h"
  48
  49 #include <apti18n.h>
  50                                                                         /*}}}*/
  51 using namespace std;
  52 FTWScanner *FTWScanner::Owner;
  53
  54 // ConfigToDoHashes - which hashes to generate                          /*{{{*/
  55 static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
  56 {
  57    if (_config->FindB(Conf, true) == true)
  58       DoHashes |= Flag;
  59    else
  60       DoHashes &= ~Flag;
  61 }
  62 static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
  63 {
  64    SingleConfigToDoHashes(DoHashes, Conf + "::MD5", Hashes::MD5SUM);
  65    SingleConfigToDoHashes(DoHashes, Conf + "::SHA1", Hashes::SHA1SUM);
  66    SingleConfigToDoHashes(DoHashes, Conf + "::SHA256", Hashes::SHA256SUM);
  67    SingleConfigToDoHashes(DoHashes, Conf + "::SHA512", Hashes::SHA512SUM);
  68 }
  69                                                                         /*}}}*/
  70
  71 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  72 // ---------------------------------------------------------------------
  73 /* */
  74 FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch): Arch(Arch), DoHashes(~0)
  75 {
  76    if (GivenOutput == NULL)
  77    {
  78       Output = new FileFd;
  79       Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
  80    }
  81    else
  82       Output = GivenOutput;
  83    ErrorPrinted = false;
  84    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  85    ConfigToDoHashes(DoHashes, "APT::FTPArchive");
  86 }
  87                                                                         /*}}}*/
  88 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* This is the FTW scanner, it processes each directory element in the
  91    directory tree. */
  92 int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
  93 {
  94    if (Flag == FTW_DNR)
  95    {
  96       Owner->NewLine(1);
  97       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  98    }
  99    if (Flag == FTW_NS)
 100    {
 101       Owner->NewLine(1);
 102       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
 103    }
 104    if (Flag != FTW_F)
 105       return 0;
 106
 107    return ScannerFile(File, true);
 108 }
 109                                                                         /*}}}*/
 110 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
 111 // ---------------------------------------------------------------------
 112 /* */
 113 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
 114 {
 115    const char *LastComponent = strrchr(File, '/');
 116    char *RealPath = NULL;
 117
 118    if (LastComponent == NULL)
 119       LastComponent = File;
 120    else
 121       LastComponent++;
 122
 123    vector<string>::const_iterator I;
 124    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 125    {
 126       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 127          break;
 128    }
 129    if (I == Owner->Patterns.end())
 130       return 0;
 131
 132    /* Process it. If the file is a link then resolve it into an absolute
 133       name.. This works best if the directory components the scanner are
 134       given are not links themselves. */
 135    char Jnk[2];
 136    Owner->OriginalPath = File;
 137    if (ReadLink &&
 138        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 139        (RealPath = realpath(File,NULL)) != 0)
 140    {
 141       Owner->DoPackage(RealPath);
 142       free(RealPath);
 143    }
 144    else
 145       Owner->DoPackage(File);
 146
 147    if (_error->empty() == false)
 148    {
 149       // Print any errors or warnings found
 150       string Err;
 151       bool SeenPath = false;
 152       while (_error->empty() == false)
 153       {
 154          Owner->NewLine(1);
 155
 156          bool const Type = _error->PopMessage(Err);
 157          if (Type == true)
 158             cerr << _("E: ") << Err << endl;
 159          else
 160             cerr << _("W: ") << Err << endl;
 161
 162          if (Err.find(File) != string::npos)
 163             SeenPath = true;
 164       }
 165
 166       if (SeenPath == false)
 167          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 168       return 0;
 169    }
 170
 171    return 0;
 172 }
 173                                                                         /*}}}*/
 174 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 175 // ---------------------------------------------------------------------
 176 /* */
 177 bool FTWScanner::RecursiveScan(string const &Dir)
 178 {
 179    char *RealPath = NULL;
 180    /* If noprefix is set then jam the scan root in, so we don't generate
 181       link followed paths out of control */
 182    if (InternalPrefix.empty() == true)
 183    {
 184       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 185          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 186       InternalPrefix = RealPath;
 187       free(RealPath);
 188    }
 189
 190    // Do recursive directory searching
 191    Owner = this;
 192    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 193
 194    // Error treewalking?
 195    if (Res != 0)
 196    {
 197       if (_error->PendingError() == false)
 198          _error->Errno("ftw",_("Tree walking failed"));
 199       return false;
 200    }
 201
 202    return true;
 203 }
 204                                                                         /*}}}*/
 205 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 206 // ---------------------------------------------------------------------
 207 /* This is an alternative to using FTW to locate files, it reads the list
 208    of files from another file. */
 209 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 210 {
 211    char *RealPath = NULL;
 212    /* If noprefix is set then jam the scan root in, so we don't generate
 213       link followed paths out of control */
 214    if (InternalPrefix.empty() == true)
 215    {
 216       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 217          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 218       InternalPrefix = RealPath;
 219       free(RealPath);
 220    }
 221
 222    Owner = this;
 223    FILE *List = fopen(File.c_str(),"r");
 224    if (List == 0)
 225       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 226
 227    /* We are a tad tricky here.. We prefix the buffer with the directory
 228       name, that way if we need a full path with just use line.. Sneaky and
 229       fully evil. */
 230    char Line[1000];
 231    char *FileStart;
 232    if (Dir.empty() == true || Dir.end()[-1] != '/')
 233       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 234    else
 235       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 236    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 237    {
 238       char *FileName = _strstrip(FileStart);
 239       if (FileName[0] == 0)
 240          continue;
 241
 242       if (FileName[0] != '/')
 243       {
 244          if (FileName != FileStart)
 245             memmove(FileStart,FileName,strlen(FileStart));
 246          FileName = Line;
 247       }
 248
 249 #if 0
 250       struct stat St;
 251       int Flag = FTW_F;
 252       if (stat(FileName,&St) != 0)
 253          Flag = FTW_NS;
 254 #endif
 255
 256       if (ScannerFile(FileName, false) != 0)
 257          break;
 258    }
 259
 260    fclose(List);
 261    return true;
 262 }
 263                                                                         /*}}}*/
 264 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 265 // ---------------------------------------------------------------------
 266 /* */
 267 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 268                         unsigned long long &DeLinkBytes,
 269                         unsigned long long const &FileSize)
 270 {
 271    // See if this isn't an internaly prefix'd file name.
 272    if (InternalPrefix.empty() == false &&
 273        InternalPrefix.length() < FileName.length() &&
 274        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 275                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 276    {
 277       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 278       {
 279          // Tidy up the display
 280          if (DeLinkBytes == 0)
 281             cout << endl;
 282
 283          NewLine(1);
 284          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 285                     SizeToStr(FileSize).c_str());
 286          c1out << flush;
 287
 288          if (NoLinkAct == false)
 289          {
 290             char OldLink[400];
 291             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 292                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 293             else
 294             {
 295                if (unlink(OriginalPath) != 0)
 296                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 297                else
 298                {
 299                   if (link(FileName.c_str(),OriginalPath) != 0)
 300                   {
 301                      // Panic! Restore the symlink
 302                      if (symlink(OldLink,OriginalPath) != 0)
 303                         _error->Errno("symlink", "failed to restore symlink");
 304                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 305                                           FileName.c_str(),
 306                                           OriginalPath);
 307                   }
 308                }
 309             }
 310          }
 311
 312          DeLinkBytes += FileSize;
 313          if (DeLinkBytes/1024 >= DeLinkLimit)
 314             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 315       }
 316
 317       FileName = OriginalPath;
 318    }
 319
 320    return true;
 321 }
 322                                                                         /*}}}*/
 323
 324 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 325 // ---------------------------------------------------------------------
 326 /* */
 327 PackagesWriter::PackagesWriter(FileFd * const GivenOutput, string const &DB,string const &Overrides,string const &ExtOverrides,
 328                                string const &Arch) :
 329    FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 330 {
 331    SetExts(".deb .udeb");
 332    DeLinkLimit = 0;
 333
 334    // Process the command line options
 335    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Packages");
 336    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 337    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 338    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 339    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 340
 341    if (Db.Loaded() == false)
 342       DoContents = false;
 343
 344    // Read the override file
 345    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 346       return;
 347    else
 348       NoOverride = true;
 349
 350    if (ExtOverrides.empty() == false)
 351       Over.ReadExtraOverride(ExtOverrides);
 352
 353    _error->DumpErrors();
 354 }
 355                                                                         /*}}}*/
 356 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 357 // ---------------------------------------------------------------------
 358 /* */
 359 bool FTWScanner::SetExts(string const &Vals)
 360 {
 361    ClearPatterns();
 362    string::size_type Start = 0;
 363    while (Start <= Vals.length()-1)
 364    {
 365       string::size_type const Space = Vals.find(' ',Start);
 366       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 367       if ( Arch.empty() == false )
 368       {
 369          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 370          AddPattern(string("*_all") + Vals.substr(Start, Length));
 371       }
 372       else
 373          AddPattern(string("*") + Vals.substr(Start, Length));
 374
 375       Start += Length + 1;
 376    }
 377
 378    return true;
 379 }
 380
 381                                                                         /*}}}*/
 382 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 383 // ---------------------------------------------------------------------
 384 /* This method takes a package and gets its control information and
 385    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 386    rewritten and the path/size/hash appended. */
 387 bool PackagesWriter::DoPackage(string FileName)
 388 {
 389    // Pull all the data we need form the DB
 390    if (Db.GetFileInfo(FileName,
 391             true, /* DoControl */
 392             DoContents,
 393             true, /* GenContentsOnly */
 394             false, /* DoSource */
 395             DoHashes, DoAlwaysStat) == false)
 396    {
 397      return false;
 398    }
 399
 400    unsigned long long FileSize = Db.GetFileSize();
 401    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 402       return false;
 403
 404    // Lookup the overide information
 405    pkgTagSection &Tags = Db.Control.Section;
 406    string Package = Tags.FindS("Package");
 407    string Architecture;
 408    // if we generate a Packages file for a given arch, we use it to
 409    // look for overrides. if we run in "simple" mode without the
 410    // "Architecures" variable in the config we use the architecure value
 411    // from the deb file
 412    if(Arch != "")
 413       Architecture = Arch;
 414    else
 415       Architecture = Tags.FindS("Architecture");
 416    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 417
 418    if (Package.empty() == true)
 419       return _error->Error(_("Archive had no package field"));
 420
 421    // If we need to do any rewriting of the header do it now..
 422    if (OverItem.get() == 0)
 423    {
 424       if (NoOverride == false)
 425       {
 426          NewLine(1);
 427          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 428       }
 429
 430       OverItem = auto_ptr<Override::Item>(new Override::Item);
 431       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 432       OverItem->Priority = Tags.FindS("Priority");
 433    }
 434
 435    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 436    string NewFileName;
 437    if (DirStrip.empty() == false &&
 438        FileName.length() > DirStrip.length() &&
 439        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 440                  DirStrip.begin(),DirStrip.end()) == 0)
 441       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 442    else
 443       NewFileName = FileName;
 444    if (PathPrefix.empty() == false)
 445       NewFileName = flCombine(PathPrefix,NewFileName);
 446
 447    /* Configuration says we don't want to include the long Description
 448       in the package file - instead we want to ship a separated file */
 449    string desc;
 450    if (LongDescription == false) {
 451       desc = Tags.FindS("Description").append("\n");
 452       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 453    }
 454
 455    // This lists all the changes to the fields we are going to make.
 456    std::vector<pkgTagSection::Tag> Changes;
 457
 458    std::string Size;
 459    strprintf(Size, "%llu", (unsigned long long) FileSize);
 460    Changes.push_back(pkgTagSection::Tag::Rewrite("Size", Size));
 461
 462    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 463    {
 464       if (hs->HashType() == "MD5Sum")
 465          Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue()));
 466       else if (hs->HashType() == "Checksum-FileSize")
 467          continue;
 468       else
 469          Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue()));
 470    }
 471    Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName));
 472    Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority));
 473    Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 474    Changes.push_back(pkgTagSection::Tag::Remove("Optional"));
 475
 476    string DescriptionMd5;
 477    if (LongDescription == false) {
 478       MD5Summation descmd5;
 479       descmd5.Add(desc.c_str());
 480       DescriptionMd5 = descmd5.Result().Value();
 481       Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5));
 482       if (TransWriter != NULL)
 483          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 484    }
 485
 486    // Rewrite the maintainer field if necessary
 487    bool MaintFailed;
 488    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 489    if (MaintFailed == true)
 490    {
 491       if (NoOverride == false)
 492       {
 493          NewLine(1);
 494          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 495                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 496       }
 497    }
 498
 499    if (NewMaint.empty() == false)
 500       Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint));
 501
 502    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 503       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 504       but dpkg does this append bit. So we do the append bit, at least that way the
 505       status file and package file will remain similar. There are other transforms
 506       but optional is the only legacy one still in use for some lazy reason. */
 507    string OptionalStr = Tags.FindS("Optional");
 508    if (OptionalStr.empty() == false)
 509    {
 510       if (Tags.FindS("Suggests").empty() == false)
 511          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 512       Changes.push_back(pkgTagSection::Tag::Rewrite("Suggests", OptionalStr));
 513    }
 514
 515    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 516         I != OverItem->FieldOverride.end(); ++I)
 517       Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 518
 519    // Rewrite and store the fields.
 520    if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false ||
 521          Output->Write("\n", 1) == false)
 522       return false;
 523
 524    return Db.Finish();
 525 }
 526                                                                         /*}}}*/
 527
 528 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 529 // ---------------------------------------------------------------------
 530 /* Create a Translation-Master file for this Packages file */
 531 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 532                                         mode_t const &Permissions) : RefCounter(0)
 533 {
 534    if (File.empty() == true)
 535       return;
 536
 537    Comp = new MultiCompress(File, TransCompress, Permissions);
 538    Output = &Comp->Input;
 539 }
 540                                                                         /*}}}*/
 541 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 542 // ---------------------------------------------------------------------
 543 /* Create a Translation-Master file for this Packages file */
 544 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 545                                   string const &MD5)
 546 {
 547    if (Output == NULL)
 548       return true;
 549
 550    // Different archs can include different versions and therefore
 551    // different descriptions - so we need to check for both name and md5.
 552    string const Record = Pkg + ":" + MD5;
 553
 554    if (Included.find(Record) != Included.end())
 555       return true;
 556
 557    std::string out;
 558    strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 559            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 560    Output->Write(out.c_str(), out.length());
 561
 562    Included.insert(Record);
 563    return true;
 564 }
 565                                                                         /*}}}*/
 566 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 567 // ---------------------------------------------------------------------
 568 /* */
 569 TranslationWriter::~TranslationWriter()
 570 {
 571    if (Comp == NULL)
 572       return;
 573
 574    delete Comp;
 575 }
 576                                                                         /*}}}*/
 577
 578 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* */
 581 SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides,
 582                              string const &ExtOverrides) :
 583    FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats)
 584 {
 585    AddPattern("*.dsc");
 586    DeLinkLimit = 0;
 587    Buffer = 0;
 588    BufSize = 0;
 589
 590    // Process the command line options
 591    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Sources");
 592    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 593    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 594
 595    // Read the override file
 596    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 597       return;
 598    else
 599       NoOverride = true;
 600
 601    // WTF?? The logic above: if we can't read binary overrides, don't even try
 602    // reading source overrides. if we can read binary overrides, then say there
 603    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 604
 605    if (ExtOverrides.empty() == false)
 606       SOver.ReadExtraOverride(ExtOverrides);
 607
 608    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 609       SOver.ReadOverride(SOverrides,true);
 610 }
 611                                                                         /*}}}*/
 612 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 613 static std::string getDscHash(unsigned int const DoHashes,
 614       Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, char const * const FieldName,
 615       HashString const * const Hash, unsigned long long Size, std::string FileName)
 616 {
 617    if ((DoHashes & DoIt) != DoIt || Tags.Exists(FieldName) == false || Hash == NULL)
 618       return "";
 619    std::ostringstream out;
 620    out << "\n " << Hash->HashValue() << " " << Size << " " << FileName
 621       << "\n " << Tags.FindS(FieldName);
 622    return out.str();
 623 }
 624 bool SourcesWriter::DoPackage(string FileName)
 625 {
 626    // Pull all the data we need form the DB
 627    if (Db.GetFileInfo(FileName,
 628             false, /* DoControl */
 629             false, /* DoContents */
 630             false, /* GenContentsOnly */
 631             true, /* DoSource */
 632             DoHashes, DoAlwaysStat) == false)
 633    {
 634       return false;
 635    }
 636
 637    // we need to perform a "write" here (this is what finish is doing)
 638    // because the call to Db.GetFileInfo() in the loop will change
 639    // the "db cursor"
 640    Db.Finish();
 641
 642    pkgTagSection Tags;
 643    if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
 644       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 645
 646    if (Tags.Exists("Source") == false)
 647       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
 648    Tags.Trim();
 649
 650    // Lookup the overide information, finding first the best priority.
 651    string BestPrio;
 652    string Bins = Tags.FindS("Binary");
 653    char Buffer[Bins.length() + 1];
 654    auto_ptr<Override::Item> OverItem(0);
 655    if (Bins.empty() == false)
 656    {
 657       strcpy(Buffer,Bins.c_str());
 658
 659       // Ignore too-long errors.
 660       char *BinList[400];
 661       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 662
 663       // Look at all the binaries
 664       unsigned char BestPrioV = pkgCache::State::Extra;
 665       for (unsigned I = 0; BinList[I] != 0; I++)
 666       {
 667          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 668          if (Itm.get() == 0)
 669             continue;
 670
 671          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 672          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 673          {
 674             BestPrioV = NewPrioV;
 675             BestPrio = Itm->Priority;
 676          }
 677
 678          if (OverItem.get() == 0)
 679             OverItem = Itm;
 680       }
 681    }
 682
 683    // If we need to do any rewriting of the header do it now..
 684    if (OverItem.get() == 0)
 685    {
 686       if (NoOverride == false)
 687       {
 688          NewLine(1);
 689          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 690       }
 691
 692       OverItem = auto_ptr<Override::Item>(new Override::Item);
 693    }
 694
 695    struct stat St;
 696    if (stat(FileName.c_str(), &St) != 0)
 697       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 698
 699    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 700    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 701    if (SOverItem.get() == 0)
 702    {
 703       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 704       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 705       if (SOverItem.get() == 0)
 706       {
 707         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 708          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 709          *SOverItem = *OverItem;
 710       }
 711    }
 712
 713    // Add the dsc to the files hash list
 714    string const strippedName = flNotDir(FileName);
 715    std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
 716    std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
 717    std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
 718    std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
 719
 720    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 721    string NewFileName;
 722    if (DirStrip.empty() == false &&
 723        FileName.length() > DirStrip.length() &&
 724        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 725       NewFileName = string(OriginalPath + DirStrip.length());
 726    else
 727       NewFileName = OriginalPath;
 728    if (PathPrefix.empty() == false)
 729       NewFileName = flCombine(PathPrefix,NewFileName);
 730
 731    string Directory = flNotFile(OriginalPath);
 732    string Package = Tags.FindS("Source");
 733
 734    // Perform operation over all of the files
 735    string ParseJnk;
 736    const char *C = Files.c_str();
 737    char *RealPath = NULL;
 738    for (;isspace(*C); C++);
 739    while (*C != 0)
 740    {
 741       // Parse each of the elements
 742       if (ParseQuoteWord(C,ParseJnk) == false ||
 743           ParseQuoteWord(C,ParseJnk) == false ||
 744           ParseQuoteWord(C,ParseJnk) == false)
 745          return _error->Error("Error parsing file record");
 746
 747       string OriginalPath = Directory + ParseJnk;
 748
 749       // Add missing hashes to source files
 750       if (((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM && !Tags.Exists("Checksums-Sha1")) ||
 751           ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM && !Tags.Exists("Checksums-Sha256")) ||
 752           ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM && !Tags.Exists("Checksums-Sha512")))
 753       {
 754          if (Db.GetFileInfo(OriginalPath,
 755                             false, /* DoControl */
 756                             false, /* DoContents */
 757                             false, /* GenContentsOnly */
 758                             false, /* DoSource */
 759                             DoHashes,
 760                             DoAlwaysStat) == false)
 761          {
 762             return _error->Error("Error getting file info");
 763          }
 764
 765          for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 766          {
 767             if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
 768                continue;
 769             char const * fieldname;
 770             std::string * out;
 771             if (hs->HashType() == "SHA1")
 772             {
 773                fieldname = "Checksums-Sha1";
 774                out = &ChecksumsSha1;
 775             }
 776             else if (hs->HashType() == "SHA256")
 777             {
 778                fieldname = "Checksums-Sha256";
 779                out = &ChecksumsSha256;
 780             }
 781             else if (hs->HashType() == "SHA512")
 782             {
 783                fieldname = "Checksums-Sha512";
 784                out = &ChecksumsSha512;
 785             }
 786             else
 787             {
 788                _error->Warning("Ignoring unknown Checksumtype %s in SourcesWriter::DoPackages", hs->HashType().c_str());
 789                continue;
 790             }
 791             if (Tags.Exists(fieldname) == true)
 792                continue;
 793             std::ostringstream streamout;
 794             streamout << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
 795             out->append(streamout.str());
 796          }
 797
 798          // write back the GetFileInfo() stats data
 799          Db.Finish();
 800       }
 801
 802       // Perform the delinking operation
 803       char Jnk[2];
 804
 805       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 806           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 807       {
 808          string RP = RealPath;
 809          free(RealPath);
 810          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 811             return false;
 812       }
 813    }
 814
 815    Directory = flNotFile(NewFileName);
 816    if (Directory.length() > 2)
 817       Directory.erase(Directory.end()-1);
 818
 819    // This lists all the changes to the fields we are going to make.
 820    // (5 hardcoded + checksums + maintainer + end marker)
 821    std::vector<pkgTagSection::Tag> Changes;
 822
 823    Changes.push_back(pkgTagSection::Tag::Remove("Source"));
 824    Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package));
 825    if (Files.empty() == false)
 826       Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files));
 827    if (ChecksumsSha1.empty() == false)
 828       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1));
 829    if (ChecksumsSha256.empty() == false)
 830       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256));
 831    if (ChecksumsSha512.empty() == false)
 832       Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512));
 833    if (Directory != "./")
 834       Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory));
 835    Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio));
 836    Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 837
 838    // Rewrite the maintainer field if necessary
 839    bool MaintFailed;
 840    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"), MaintFailed);
 841    if (MaintFailed == true)
 842    {
 843       if (NoOverride == false)
 844       {
 845          NewLine(1);
 846          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 847                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 848       }
 849    }
 850    if (NewMaint.empty() == false)
 851       Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str()));
 852
 853    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 854         I != SOverItem->FieldOverride.end(); ++I)
 855       Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 856
 857    // Rewrite and store the fields.
 858    if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false ||
 859          Output->Write("\n", 1) == false)
 860       return false;
 861
 862    Stats.Packages++;
 863
 864    return true;
 865 }
 866                                                                         /*}}}*/
 867
 868 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 869 // ---------------------------------------------------------------------
 870 /* */
 871 ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB, string const &Arch) :
 872                     FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats)
 873
 874 {
 875    SetExts(".deb");
 876 }
 877                                                                         /*}}}*/
 878 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 879 // ---------------------------------------------------------------------
 880 /* If Package is the empty string the control record will be parsed to
 881    determine what the package name is. */
 882 bool ContentsWriter::DoPackage(string FileName, string Package)
 883 {
 884    if (!Db.GetFileInfo(FileName,
 885             Package.empty(), /* DoControl */
 886             true, /* DoContents */
 887             false, /* GenContentsOnly */
 888             false, /* DoSource */
 889             0, /* DoHashes */
 890             false /* checkMtime */))
 891    {
 892       return false;
 893    }
 894
 895    // Parse the package name
 896    if (Package.empty() == true)
 897    {
 898       Package = Db.Control.Section.FindS("Package");
 899    }
 900
 901    Db.Contents.Add(Gen,Package);
 902
 903    return Db.Finish();
 904 }
 905                                                                         /*}}}*/
 906 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 907 // ---------------------------------------------------------------------
 908 /* */
 909 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 910 {
 911    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 912    if (_error->PendingError() == true)
 913       return false;
 914
 915    // Open the package file
 916    FileFd Fd;
 917    if (Pkgs.OpenOld(Fd) == false)
 918       return false;
 919
 920    pkgTagFile Tags(&Fd);
 921    if (_error->PendingError() == true)
 922       return false;
 923
 924    // Parse.
 925    pkgTagSection Section;
 926    while (Tags.Step(Section) == true)
 927    {
 928       string File = flCombine(Prefix,Section.FindS("FileName"));
 929       string Package = Section.FindS("Section");
 930       if (Package.empty() == false && Package.end()[-1] != '/')
 931       {
 932          Package += '/';
 933          Package += Section.FindS("Package");
 934       }
 935       else
 936          Package += Section.FindS("Package");
 937
 938       DoPackage(File,Package);
 939       if (_error->empty() == false)
 940       {
 941          _error->Error("Errors apply to file '%s'",File.c_str());
 942          _error->DumpErrors();
 943       }
 944    }
 945
 946    // Tidy the compressor
 947    Fd.Close();
 948
 949    return true;
 950 }
 951
 952                                                                         /*}}}*/
 953
 954 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 955 // ---------------------------------------------------------------------
 956 /* */
 957 ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput)
 958 {
 959    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 960    {
 961       AddPattern("Packages");
 962       AddPattern("Packages.gz");
 963       AddPattern("Packages.bz2");
 964       AddPattern("Packages.lzma");
 965       AddPattern("Packages.xz");
 966       AddPattern("Translation-*");
 967       AddPattern("Sources");
 968       AddPattern("Sources.gz");
 969       AddPattern("Sources.bz2");
 970       AddPattern("Sources.lzma");
 971       AddPattern("Sources.xz");
 972       AddPattern("Release");
 973       AddPattern("Contents-*");
 974       AddPattern("Index");
 975       AddPattern("md5sum.txt");
 976    }
 977    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 978
 979    time_t const now = time(NULL);
 980
 981    setlocale(LC_TIME, "C");
 982
 983    char datestr[128];
 984    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 985                 gmtime(&now)) == 0)
 986    {
 987       datestr[0] = '\0';
 988    }
 989
 990    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 991    char validstr[128];
 992    if (now == validuntil ||
 993        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 994                 gmtime(&validuntil)) == 0)
 995    {
 996       validstr[0] = '\0';
 997    }
 998
 999    setlocale(LC_TIME, "");
1000
1001    map<string,string> Fields;
1002    Fields["Origin"] = "";
1003    Fields["Label"] = "";
1004    Fields["Suite"] = "";
1005    Fields["Version"] = "";
1006    Fields["Codename"] = "";
1007    Fields["Date"] = datestr;
1008    Fields["Valid-Until"] = validstr;
1009    Fields["Architectures"] = "";
1010    Fields["Components"] = "";
1011    Fields["Description"] = "";
1012
1013    for(map<string,string>::const_iterator I = Fields.begin();
1014        I != Fields.end();
1015        ++I)
1016    {
1017       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1018       string Value = _config->Find(Config, (*I).second.c_str());
1019       if (Value == "")
1020          continue;
1021
1022       std::string const out = I->first + ": " + Value + "\n";
1023       Output->Write(out.c_str(), out.length());
1024    }
1025
1026    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
1027 }
1028                                                                         /*}}}*/
1029 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1030 // ---------------------------------------------------------------------
1031 bool ReleaseWriter::DoPackage(string FileName)
1032 {
1033    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1034    string NewFileName;
1035    if (DirStrip.empty() == false &&
1036        FileName.length() > DirStrip.length() &&
1037        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1038                  DirStrip.begin(),DirStrip.end()) == 0)
1039    {
1040       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1041       while (NewFileName[0] == '/')
1042          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1043    }
1044    else
1045       NewFileName = FileName;
1046
1047    if (PathPrefix.empty() == false)
1048       NewFileName = flCombine(PathPrefix,NewFileName);
1049
1050    FileFd fd(FileName, FileFd::ReadOnly);
1051
1052    if (!fd.IsOpen())
1053    {
1054       return false;
1055    }
1056
1057    CheckSums[NewFileName].size = fd.Size();
1058
1059    Hashes hs(DoHashes);
1060    hs.AddFD(fd);
1061    CheckSums[NewFileName].Hashes = hs.GetHashStringList();
1062    fd.Close();
1063
1064    return true;
1065 }
1066
1067                                                                         /*}}}*/
1068 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1069 // ---------------------------------------------------------------------
1070 static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map<string, ReleaseWriter::CheckSum> const &CheckSums)
1071 {
1072    {
1073       std::string out;
1074       strprintf(out, "%s:\n", Type);
1075       Output.Write(out.c_str(), out.length());
1076    }
1077    for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
1078          I != CheckSums.end(); ++I)
1079    {
1080       HashString const * const hs = I->second.Hashes.find(Type);
1081       if (hs == NULL)
1082          continue;
1083       std::string out;
1084       strprintf(out, " %s %16llu %s\n",
1085             hs->HashValue().c_str(),
1086             (*I).second.size,
1087             (*I).first.c_str());
1088       Output.Write(out.c_str(), out.length());
1089    }
1090 }
1091 void ReleaseWriter::Finish()
1092 {
1093    if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
1094       printChecksumTypeRecord(*Output, "MD5Sum", CheckSums);
1095    if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
1096       printChecksumTypeRecord(*Output, "SHA1", CheckSums);
1097    if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
1098       printChecksumTypeRecord(*Output, "SHA256", CheckSums);
1099    if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
1100       printChecksumTypeRecord(*Output, "SHA512", CheckSums);
1101 }