ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/deblistparser.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/fileutl.h>
  20 #include <apt-pkg/gpgv.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/debfile.h>
  25 #include <apt-pkg/pkgcache.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/sha2.h>
  28 #include <apt-pkg/tagfile.h>
  29
  30 #include <ctype.h>
  31 #include <fnmatch.h>
  32 #include <ftw.h>
  33 #include <locale.h>
  34 #include <string.h>
  35 #include <sys/stat.h>
  36 #include <sys/types.h>
  37 #include <unistd.h>
  38 #include <ctime>
  39 #include <iostream>
  40 #include <sstream>
  41 #include <memory>
  42 #include <utility>
  43
  44 #include "apt-ftparchive.h"
  45 #include "writer.h"
  46 #include "cachedb.h"
  47 #include "multicompress.h"
  48
  49 #include <apti18n.h>
  50                                                                         /*}}}*/
  51 using namespace std;
  52 FTWScanner *FTWScanner::Owner;
  53
  54 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 static inline TFRewriteData SetTFRewriteData(const char *tag,
  58                              const char *rewrite,
  59                              const char *newtag = 0)
  60 {
  61    TFRewriteData tfrd;
  62    tfrd.Tag = tag;
  63    tfrd.Rewrite = rewrite;
  64    tfrd.NewTag = newtag;
  65    return tfrd;
  66 }
  67                                                                         /*}}}*/
  68 // ConfigToDoHashes - which hashes to generate                          /*{{{*/
  69 static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
  70 {
  71    if (_config->FindB(Conf, true) == true)
  72       DoHashes |= Flag;
  73    else
  74       DoHashes &= ~Flag;
  75 }
  76 static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
  77 {
  78    SingleConfigToDoHashes(DoHashes, Conf + "::MD5", Hashes::MD5SUM);
  79    SingleConfigToDoHashes(DoHashes, Conf + "::SHA1", Hashes::SHA1SUM);
  80    SingleConfigToDoHashes(DoHashes, Conf + "::SHA256", Hashes::SHA256SUM);
  81    SingleConfigToDoHashes(DoHashes, Conf + "::SHA512", Hashes::SHA512SUM);
  82 }
  83                                                                         /*}}}*/
  84
  85 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* */
  88 FTWScanner::FTWScanner(string const &Arch): Arch(Arch), DoHashes(~0)
  89 {
  90    ErrorPrinted = false;
  91    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  92    ConfigToDoHashes(DoHashes, "APT::FTPArchive");
  93 }
  94                                                                         /*}}}*/
  95 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* This is the FTW scanner, it processes each directory element in the
  98    directory tree. */
  99 int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
 100 {
 101    if (Flag == FTW_DNR)
 102    {
 103       Owner->NewLine(1);
 104       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
 105    }
 106    if (Flag == FTW_NS)
 107    {
 108       Owner->NewLine(1);
 109       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
 110    }
 111    if (Flag != FTW_F)
 112       return 0;
 113
 114    return ScannerFile(File, true);
 115 }
 116                                                                         /*}}}*/
 117 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
 121 {
 122    const char *LastComponent = strrchr(File, '/');
 123    char *RealPath = NULL;
 124
 125    if (LastComponent == NULL)
 126       LastComponent = File;
 127    else
 128       LastComponent++;
 129
 130    vector<string>::const_iterator I;
 131    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 132    {
 133       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 134          break;
 135    }
 136    if (I == Owner->Patterns.end())
 137       return 0;
 138
 139    /* Process it. If the file is a link then resolve it into an absolute
 140       name.. This works best if the directory components the scanner are
 141       given are not links themselves. */
 142    char Jnk[2];
 143    Owner->OriginalPath = File;
 144    if (ReadLink &&
 145        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 146        (RealPath = realpath(File,NULL)) != 0)
 147    {
 148       Owner->DoPackage(RealPath);
 149       free(RealPath);
 150    }
 151    else
 152       Owner->DoPackage(File);
 153
 154    if (_error->empty() == false)
 155    {
 156       // Print any errors or warnings found
 157       string Err;
 158       bool SeenPath = false;
 159       while (_error->empty() == false)
 160       {
 161          Owner->NewLine(1);
 162
 163          bool const Type = _error->PopMessage(Err);
 164          if (Type == true)
 165             cerr << _("E: ") << Err << endl;
 166          else
 167             cerr << _("W: ") << Err << endl;
 168
 169          if (Err.find(File) != string::npos)
 170             SeenPath = true;
 171       }
 172
 173       if (SeenPath == false)
 174          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 175       return 0;
 176    }
 177
 178    return 0;
 179 }
 180                                                                         /*}}}*/
 181 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 182 // ---------------------------------------------------------------------
 183 /* */
 184 bool FTWScanner::RecursiveScan(string const &Dir)
 185 {
 186    char *RealPath = NULL;
 187    /* If noprefix is set then jam the scan root in, so we don't generate
 188       link followed paths out of control */
 189    if (InternalPrefix.empty() == true)
 190    {
 191       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 192          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 193       InternalPrefix = RealPath;
 194       free(RealPath);
 195    }
 196
 197    // Do recursive directory searching
 198    Owner = this;
 199    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 200
 201    // Error treewalking?
 202    if (Res != 0)
 203    {
 204       if (_error->PendingError() == false)
 205          _error->Errno("ftw",_("Tree walking failed"));
 206       return false;
 207    }
 208
 209    return true;
 210 }
 211                                                                         /*}}}*/
 212 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 213 // ---------------------------------------------------------------------
 214 /* This is an alternative to using FTW to locate files, it reads the list
 215    of files from another file. */
 216 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 217 {
 218    char *RealPath = NULL;
 219    /* If noprefix is set then jam the scan root in, so we don't generate
 220       link followed paths out of control */
 221    if (InternalPrefix.empty() == true)
 222    {
 223       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 224          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 225       InternalPrefix = RealPath;
 226       free(RealPath);
 227    }
 228
 229    Owner = this;
 230    FILE *List = fopen(File.c_str(),"r");
 231    if (List == 0)
 232       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 233
 234    /* We are a tad tricky here.. We prefix the buffer with the directory
 235       name, that way if we need a full path with just use line.. Sneaky and
 236       fully evil. */
 237    char Line[1000];
 238    char *FileStart;
 239    if (Dir.empty() == true || Dir.end()[-1] != '/')
 240       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 241    else
 242       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 243    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 244    {
 245       char *FileName = _strstrip(FileStart);
 246       if (FileName[0] == 0)
 247          continue;
 248
 249       if (FileName[0] != '/')
 250       {
 251          if (FileName != FileStart)
 252             memmove(FileStart,FileName,strlen(FileStart));
 253          FileName = Line;
 254       }
 255
 256 #if 0
 257       struct stat St;
 258       int Flag = FTW_F;
 259       if (stat(FileName,&St) != 0)
 260          Flag = FTW_NS;
 261 #endif
 262
 263       if (ScannerFile(FileName, false) != 0)
 264          break;
 265    }
 266
 267    fclose(List);
 268    return true;
 269 }
 270                                                                         /*}}}*/
 271 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 272 // ---------------------------------------------------------------------
 273 /* */
 274 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 275                         unsigned long long &DeLinkBytes,
 276                         unsigned long long const &FileSize)
 277 {
 278    // See if this isn't an internaly prefix'd file name.
 279    if (InternalPrefix.empty() == false &&
 280        InternalPrefix.length() < FileName.length() &&
 281        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 282                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 283    {
 284       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 285       {
 286          // Tidy up the display
 287          if (DeLinkBytes == 0)
 288             cout << endl;
 289
 290          NewLine(1);
 291          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 292                     SizeToStr(FileSize).c_str());
 293          c1out << flush;
 294
 295          if (NoLinkAct == false)
 296          {
 297             char OldLink[400];
 298             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 299                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 300             else
 301             {
 302                if (unlink(OriginalPath) != 0)
 303                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 304                else
 305                {
 306                   if (link(FileName.c_str(),OriginalPath) != 0)
 307                   {
 308                      // Panic! Restore the symlink
 309                      if (symlink(OldLink,OriginalPath) != 0)
 310                         _error->Errno("symlink", "failed to restore symlink");
 311                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 312                                           FileName.c_str(),
 313                                           OriginalPath);
 314                   }
 315                }
 316             }
 317          }
 318
 319          DeLinkBytes += FileSize;
 320          if (DeLinkBytes/1024 >= DeLinkLimit)
 321             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 322       }
 323
 324       FileName = OriginalPath;
 325    }
 326
 327    return true;
 328 }
 329                                                                         /*}}}*/
 330
 331 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* */
 334 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 335                                string const &Arch) :
 336    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 337 {
 338    Output = stdout;
 339    SetExts(".deb .udeb");
 340    DeLinkLimit = 0;
 341
 342    // Process the command line options
 343    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Packages");
 344    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 345    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 346    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 347    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 348
 349    if (Db.Loaded() == false)
 350       DoContents = false;
 351
 352    // Read the override file
 353    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 354       return;
 355    else
 356       NoOverride = true;
 357
 358    if (ExtOverrides.empty() == false)
 359       Over.ReadExtraOverride(ExtOverrides);
 360
 361    _error->DumpErrors();
 362 }
 363                                                                         /*}}}*/
 364 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* */
 367 bool FTWScanner::SetExts(string const &Vals)
 368 {
 369    ClearPatterns();
 370    string::size_type Start = 0;
 371    while (Start <= Vals.length()-1)
 372    {
 373       string::size_type const Space = Vals.find(' ',Start);
 374       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 375       if ( Arch.empty() == false )
 376       {
 377          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 378          AddPattern(string("*_all") + Vals.substr(Start, Length));
 379       }
 380       else
 381          AddPattern(string("*") + Vals.substr(Start, Length));
 382
 383       Start += Length + 1;
 384    }
 385
 386    return true;
 387 }
 388
 389                                                                         /*}}}*/
 390 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 391 // ---------------------------------------------------------------------
 392 /* This method takes a package and gets its control information and
 393    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 394    rewritten and the path/size/hash appended. */
 395 bool PackagesWriter::DoPackage(string FileName)
 396 {
 397    // Pull all the data we need form the DB
 398    if (Db.GetFileInfo(FileName,
 399             true, /* DoControl */
 400             DoContents,
 401             true, /* GenContentsOnly */
 402             false, /* DoSource */
 403             DoHashes, DoAlwaysStat) == false)
 404    {
 405      return false;
 406    }
 407
 408    unsigned long long FileSize = Db.GetFileSize();
 409    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 410       return false;
 411
 412    // Lookup the overide information
 413    pkgTagSection &Tags = Db.Control.Section;
 414    string Package = Tags.FindS("Package");
 415    string Architecture;
 416    // if we generate a Packages file for a given arch, we use it to
 417    // look for overrides. if we run in "simple" mode without the
 418    // "Architecures" variable in the config we use the architecure value
 419    // from the deb file
 420    if(Arch != "")
 421       Architecture = Arch;
 422    else
 423       Architecture = Tags.FindS("Architecture");
 424    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 425
 426    if (Package.empty() == true)
 427       return _error->Error(_("Archive had no package field"));
 428
 429    // If we need to do any rewriting of the header do it now..
 430    if (OverItem.get() == 0)
 431    {
 432       if (NoOverride == false)
 433       {
 434          NewLine(1);
 435          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 436       }
 437
 438       OverItem = auto_ptr<Override::Item>(new Override::Item);
 439       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 440       OverItem->Priority = Tags.FindS("Priority");
 441    }
 442
 443    char Size[40];
 444    sprintf(Size,"%llu", (unsigned long long) FileSize);
 445
 446    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 447    string NewFileName;
 448    if (DirStrip.empty() == false &&
 449        FileName.length() > DirStrip.length() &&
 450        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 451                  DirStrip.begin(),DirStrip.end()) == 0)
 452       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 453    else
 454       NewFileName = FileName;
 455    if (PathPrefix.empty() == false)
 456       NewFileName = flCombine(PathPrefix,NewFileName);
 457
 458    /* Configuration says we don't want to include the long Description
 459       in the package file - instead we want to ship a separated file */
 460    string desc;
 461    if (LongDescription == false) {
 462       desc = Tags.FindS("Description").append("\n");
 463       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 464    }
 465
 466    // This lists all the changes to the fields we are going to make.
 467    std::vector<TFRewriteData> Changes;
 468
 469    Changes.push_back(SetTFRewriteData("Size", Size));
 470    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 471    {
 472       if (hs->HashType() == "MD5Sum")
 473          Changes.push_back(SetTFRewriteData("MD5sum", hs->HashValue().c_str()));
 474       else if (hs->HashType() == "Checksum-FileSize")
 475          continue;
 476       else
 477          Changes.push_back(SetTFRewriteData(hs->HashType().c_str(), hs->HashValue().c_str()));
 478    }
 479    Changes.push_back(SetTFRewriteData("Filename", NewFileName.c_str()));
 480    Changes.push_back(SetTFRewriteData("Priority", OverItem->Priority.c_str()));
 481    Changes.push_back(SetTFRewriteData("Status", 0));
 482    Changes.push_back(SetTFRewriteData("Optional", 0));
 483
 484    string DescriptionMd5;
 485    if (LongDescription == false) {
 486       MD5Summation descmd5;
 487       descmd5.Add(desc.c_str());
 488       DescriptionMd5 = descmd5.Result().Value();
 489       Changes.push_back(SetTFRewriteData("Description-md5", DescriptionMd5.c_str()));
 490       if (TransWriter != NULL)
 491          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 492    }
 493
 494    // Rewrite the maintainer field if necessary
 495    bool MaintFailed;
 496    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 497    if (MaintFailed == true)
 498    {
 499       if (NoOverride == false)
 500       {
 501          NewLine(1);
 502          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 503                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 504       }
 505    }
 506
 507    if (NewMaint.empty() == false)
 508       Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
 509
 510    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 511       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 512       but dpkg does this append bit. So we do the append bit, at least that way the
 513       status file and package file will remain similar. There are other transforms
 514       but optional is the only legacy one still in use for some lazy reason. */
 515    string OptionalStr = Tags.FindS("Optional");
 516    if (OptionalStr.empty() == false)
 517    {
 518       if (Tags.FindS("Suggests").empty() == false)
 519          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 520       Changes.push_back(SetTFRewriteData("Suggests", OptionalStr.c_str()));
 521    }
 522
 523    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 524         I != OverItem->FieldOverride.end(); ++I)
 525       Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
 526
 527    Changes.push_back(SetTFRewriteData( 0, 0));
 528
 529    // Rewrite and store the fields.
 530    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes.data()) == false)
 531       return false;
 532    fprintf(Output,"\n");
 533
 534    return Db.Finish();
 535 }
 536                                                                         /*}}}*/
 537
 538 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 539 // ---------------------------------------------------------------------
 540 /* Create a Translation-Master file for this Packages file */
 541 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 542                                         mode_t const &Permissions) : Output(NULL),
 543                                                         RefCounter(0)
 544 {
 545    if (File.empty() == true)
 546       return;
 547
 548    Comp = new MultiCompress(File, TransCompress, Permissions);
 549    Output = Comp->Input;
 550 }
 551                                                                         /*}}}*/
 552 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* Create a Translation-Master file for this Packages file */
 555 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 556                                   string const &MD5)
 557 {
 558    if (Output == NULL)
 559       return true;
 560
 561    // Different archs can include different versions and therefore
 562    // different descriptions - so we need to check for both name and md5.
 563    string const Record = Pkg + ":" + MD5;
 564
 565    if (Included.find(Record) != Included.end())
 566       return true;
 567
 568    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 569            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 570
 571    Included.insert(Record);
 572    return true;
 573 }
 574                                                                         /*}}}*/
 575 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 576 // ---------------------------------------------------------------------
 577 /* */
 578 TranslationWriter::~TranslationWriter()
 579 {
 580    if (Comp == NULL)
 581       return;
 582
 583    delete Comp;
 584 }
 585                                                                         /*}}}*/
 586
 587 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 588 // ---------------------------------------------------------------------
 589 /* */
 590 SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
 591                              string const &ExtOverrides) :
 592    Db(DB), Stats(Db.Stats)
 593 {
 594    Output = stdout;
 595    AddPattern("*.dsc");
 596    DeLinkLimit = 0;
 597    Buffer = 0;
 598    BufSize = 0;
 599
 600    // Process the command line options
 601    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Sources");
 602    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 603    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 604
 605    // Read the override file
 606    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 607       return;
 608    else
 609       NoOverride = true;
 610
 611    // WTF?? The logic above: if we can't read binary overrides, don't even try
 612    // reading source overrides. if we can read binary overrides, then say there
 613    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 614
 615    if (ExtOverrides.empty() == false)
 616       SOver.ReadExtraOverride(ExtOverrides);
 617
 618    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 619       SOver.ReadOverride(SOverrides,true);
 620 }
 621                                                                         /*}}}*/
 622 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 623 static std::ostream& addDscHash(std::ostream &out, unsigned int const DoHashes,
 624       Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, char const * const FieldName,
 625       HashString const * const Hash, unsigned long long Size, std::string FileName)
 626 {
 627    if ((DoHashes & DoIt) != DoIt || Tags.Exists(FieldName) == false || Hash == NULL)
 628       return out;
 629    out << "\n " << Hash->HashValue() << " " << Size << " " << FileName
 630       << "\n " << Tags.FindS(FieldName);
 631    return out;
 632 }
 633 bool SourcesWriter::DoPackage(string FileName)
 634 {
 635    // Pull all the data we need form the DB
 636    if (Db.GetFileInfo(FileName,
 637             false, /* DoControl */
 638             false, /* DoContents */
 639             false, /* GenContentsOnly */
 640             true, /* DoSource */
 641             DoHashes, DoAlwaysStat) == false)
 642    {
 643       return false;
 644    }
 645
 646    // we need to perform a "write" here (this is what finish is doing)
 647    // because the call to Db.GetFileInfo() in the loop will change
 648    // the "db cursor"
 649    Db.Finish();
 650
 651    pkgTagSection Tags;
 652    if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
 653       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 654
 655    if (Tags.Exists("Source") == false)
 656       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
 657    Tags.Trim();
 658
 659    // Lookup the overide information, finding first the best priority.
 660    string BestPrio;
 661    string Bins = Tags.FindS("Binary");
 662    char Buffer[Bins.length() + 1];
 663    auto_ptr<Override::Item> OverItem(0);
 664    if (Bins.empty() == false)
 665    {
 666       strcpy(Buffer,Bins.c_str());
 667
 668       // Ignore too-long errors.
 669       char *BinList[400];
 670       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 671
 672       // Look at all the binaries
 673       unsigned char BestPrioV = pkgCache::State::Extra;
 674       for (unsigned I = 0; BinList[I] != 0; I++)
 675       {
 676          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 677          if (Itm.get() == 0)
 678             continue;
 679
 680          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 681          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 682          {
 683             BestPrioV = NewPrioV;
 684             BestPrio = Itm->Priority;
 685          }
 686
 687          if (OverItem.get() == 0)
 688             OverItem = Itm;
 689       }
 690    }
 691
 692    // If we need to do any rewriting of the header do it now..
 693    if (OverItem.get() == 0)
 694    {
 695       if (NoOverride == false)
 696       {
 697          NewLine(1);
 698          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 699       }
 700
 701       OverItem = auto_ptr<Override::Item>(new Override::Item);
 702    }
 703
 704    struct stat St;
 705    if (stat(FileName.c_str(), &St) != 0)
 706       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 707
 708    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 709    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 710    if (SOverItem.get() == 0)
 711    {
 712       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 713       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 714       if (SOverItem.get() == 0)
 715       {
 716         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 717          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 718          *SOverItem = *OverItem;
 719       }
 720    }
 721
 722    // Add the dsc to the files hash list
 723    string const strippedName = flNotDir(FileName);
 724    std::ostringstream ostreamFiles;
 725    addDscHash(ostreamFiles, DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
 726    string const Files = ostreamFiles.str();
 727
 728    std::ostringstream ostreamSha1;
 729    addDscHash(ostreamSha1, DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
 730    std::ostringstream ostreamSha256;
 731    addDscHash(ostreamSha256, DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
 732    std::ostringstream ostreamSha512;
 733    addDscHash(ostreamSha512, DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
 734
 735    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 736    string NewFileName;
 737    if (DirStrip.empty() == false &&
 738        FileName.length() > DirStrip.length() &&
 739        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 740       NewFileName = string(OriginalPath + DirStrip.length());
 741    else
 742       NewFileName = OriginalPath;
 743    if (PathPrefix.empty() == false)
 744       NewFileName = flCombine(PathPrefix,NewFileName);
 745
 746    string Directory = flNotFile(OriginalPath);
 747    string Package = Tags.FindS("Source");
 748
 749    // Perform operation over all of the files
 750    string ParseJnk;
 751    const char *C = Files.c_str();
 752    char *RealPath = NULL;
 753    for (;isspace(*C); C++);
 754    while (*C != 0)
 755    {
 756       // Parse each of the elements
 757       if (ParseQuoteWord(C,ParseJnk) == false ||
 758           ParseQuoteWord(C,ParseJnk) == false ||
 759           ParseQuoteWord(C,ParseJnk) == false)
 760          return _error->Error("Error parsing file record");
 761
 762       string OriginalPath = Directory + ParseJnk;
 763
 764       // Add missing hashes to source files
 765       if (((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM && !Tags.Exists("Checksums-Sha1")) ||
 766           ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM && !Tags.Exists("Checksums-Sha256")) ||
 767           ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM && !Tags.Exists("Checksums-Sha512")))
 768       {
 769          if (Db.GetFileInfo(OriginalPath,
 770                             false, /* DoControl */
 771                             false, /* DoContents */
 772                             false, /* GenContentsOnly */
 773                             false, /* DoSource */
 774                             DoHashes,
 775                             DoAlwaysStat) == false)
 776          {
 777             return _error->Error("Error getting file info");
 778          }
 779
 780          for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 781          {
 782             if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
 783                continue;
 784             char const * fieldname;
 785             std::ostream * out;
 786             if (hs->HashType() == "SHA1")
 787             {
 788                fieldname = "Checksums-Sha1";
 789                out = &ostreamSha1;
 790             }
 791             else if (hs->HashType() == "SHA256")
 792             {
 793                fieldname = "Checksums-Sha256";
 794                out = &ostreamSha256;
 795             }
 796             else if (hs->HashType() == "SHA512")
 797             {
 798                fieldname = "Checksums-Sha512";
 799                out = &ostreamSha512;
 800             }
 801             else
 802             {
 803                _error->Warning("Ignoring unknown Checksumtype %s in SourcesWriter::DoPackages", hs->HashType().c_str());
 804                continue;
 805             }
 806             if (Tags.Exists(fieldname) == true)
 807                continue;
 808             (*out) << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
 809          }
 810
 811          // write back the GetFileInfo() stats data
 812          Db.Finish();
 813       }
 814
 815       // Perform the delinking operation
 816       char Jnk[2];
 817
 818       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 819           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 820       {
 821          string RP = RealPath;
 822          free(RealPath);
 823          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 824             return false;
 825       }
 826    }
 827
 828    Directory = flNotFile(NewFileName);
 829    if (Directory.length() > 2)
 830       Directory.erase(Directory.end()-1);
 831
 832    string const ChecksumsSha1 = ostreamSha1.str();
 833    string const ChecksumsSha256 = ostreamSha256.str();
 834    string const ChecksumsSha512 = ostreamSha512.str();
 835
 836    // This lists all the changes to the fields we are going to make.
 837    // (5 hardcoded + checksums + maintainer + end marker)
 838    std::vector<TFRewriteData> Changes;
 839
 840    Changes.push_back(SetTFRewriteData("Source",Package.c_str(),"Package"));
 841    if (Files.empty() == false)
 842       Changes.push_back(SetTFRewriteData("Files",Files.c_str()));
 843    if (ChecksumsSha1.empty() == false)
 844       Changes.push_back(SetTFRewriteData("Checksums-Sha1",ChecksumsSha1.c_str()));
 845    if (ChecksumsSha256.empty() == false)
 846       Changes.push_back(SetTFRewriteData("Checksums-Sha256",ChecksumsSha256.c_str()));
 847    if (ChecksumsSha512.empty() == false)
 848       Changes.push_back(SetTFRewriteData("Checksums-Sha512",ChecksumsSha512.c_str()));
 849    if (Directory != "./")
 850       Changes.push_back(SetTFRewriteData("Directory",Directory.c_str()));
 851    Changes.push_back(SetTFRewriteData("Priority",BestPrio.c_str()));
 852    Changes.push_back(SetTFRewriteData("Status",0));
 853
 854    // Rewrite the maintainer field if necessary
 855    bool MaintFailed;
 856    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 857    if (MaintFailed == true)
 858    {
 859       if (NoOverride == false)
 860       {
 861          NewLine(1);
 862          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 863                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 864       }
 865    }
 866    if (NewMaint.empty() == false)
 867       Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
 868
 869    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 870         I != SOverItem->FieldOverride.end(); ++I)
 871       Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
 872
 873    Changes.push_back(SetTFRewriteData(0, 0));
 874
 875    // Rewrite and store the fields.
 876    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes.data()) == false)
 877       return false;
 878    fprintf(Output,"\n");
 879
 880    Stats.Packages++;
 881
 882    return true;
 883 }
 884                                                                         /*}}}*/
 885
 886 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 887 // ---------------------------------------------------------------------
 888 /* */
 889 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 890                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 891
 892 {
 893    SetExts(".deb");
 894    Output = stdout;
 895 }
 896                                                                         /*}}}*/
 897 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 898 // ---------------------------------------------------------------------
 899 /* If Package is the empty string the control record will be parsed to
 900    determine what the package name is. */
 901 bool ContentsWriter::DoPackage(string FileName, string Package)
 902 {
 903    if (!Db.GetFileInfo(FileName,
 904             Package.empty(), /* DoControl */
 905             true, /* DoContents */
 906             false, /* GenContentsOnly */
 907             false, /* DoSource */
 908             0, /* DoHashes */
 909             false /* checkMtime */))
 910    {
 911       return false;
 912    }
 913
 914    // Parse the package name
 915    if (Package.empty() == true)
 916    {
 917       Package = Db.Control.Section.FindS("Package");
 918    }
 919
 920    Db.Contents.Add(Gen,Package);
 921
 922    return Db.Finish();
 923 }
 924                                                                         /*}}}*/
 925 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 926 // ---------------------------------------------------------------------
 927 /* */
 928 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 929 {
 930    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 931    if (_error->PendingError() == true)
 932       return false;
 933
 934    // Open the package file
 935    FileFd Fd;
 936    if (Pkgs.OpenOld(Fd) == false)
 937       return false;
 938
 939    pkgTagFile Tags(&Fd);
 940    if (_error->PendingError() == true)
 941       return false;
 942
 943    // Parse.
 944    pkgTagSection Section;
 945    while (Tags.Step(Section) == true)
 946    {
 947       string File = flCombine(Prefix,Section.FindS("FileName"));
 948       string Package = Section.FindS("Section");
 949       if (Package.empty() == false && Package.end()[-1] != '/')
 950       {
 951          Package += '/';
 952          Package += Section.FindS("Package");
 953       }
 954       else
 955          Package += Section.FindS("Package");
 956
 957       DoPackage(File,Package);
 958       if (_error->empty() == false)
 959       {
 960          _error->Error("Errors apply to file '%s'",File.c_str());
 961          _error->DumpErrors();
 962       }
 963    }
 964
 965    // Tidy the compressor
 966    Fd.Close();
 967
 968    return true;
 969 }
 970
 971                                                                         /*}}}*/
 972
 973 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 974 // ---------------------------------------------------------------------
 975 /* */
 976 ReleaseWriter::ReleaseWriter(string const &/*DB*/)
 977 {
 978    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 979    {
 980       AddPattern("Packages");
 981       AddPattern("Packages.gz");
 982       AddPattern("Packages.bz2");
 983       AddPattern("Packages.lzma");
 984       AddPattern("Packages.xz");
 985       AddPattern("Translation-*");
 986       AddPattern("Sources");
 987       AddPattern("Sources.gz");
 988       AddPattern("Sources.bz2");
 989       AddPattern("Sources.lzma");
 990       AddPattern("Sources.xz");
 991       AddPattern("Release");
 992       AddPattern("Contents-*");
 993       AddPattern("Index");
 994       AddPattern("md5sum.txt");
 995    }
 996    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 997
 998    Output = stdout;
 999    time_t const now = time(NULL);
1000
1001    setlocale(LC_TIME, "C");
1002
1003    char datestr[128];
1004    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
1005                 gmtime(&now)) == 0)
1006    {
1007       datestr[0] = '\0';
1008    }
1009
1010    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
1011    char validstr[128];
1012    if (now == validuntil ||
1013        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
1014                 gmtime(&validuntil)) == 0)
1015    {
1016       validstr[0] = '\0';
1017    }
1018
1019    setlocale(LC_TIME, "");
1020
1021    map<string,string> Fields;
1022    Fields["Origin"] = "";
1023    Fields["Label"] = "";
1024    Fields["Suite"] = "";
1025    Fields["Version"] = "";
1026    Fields["Codename"] = "";
1027    Fields["Date"] = datestr;
1028    Fields["Valid-Until"] = validstr;
1029    Fields["Architectures"] = "";
1030    Fields["Components"] = "";
1031    Fields["Description"] = "";
1032
1033    for(map<string,string>::const_iterator I = Fields.begin();
1034        I != Fields.end();
1035        ++I)
1036    {
1037       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1038       string Value = _config->Find(Config, (*I).second.c_str());
1039       if (Value == "")
1040          continue;
1041
1042       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1043    }
1044
1045    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
1046 }
1047                                                                         /*}}}*/
1048 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1049 // ---------------------------------------------------------------------
1050 bool ReleaseWriter::DoPackage(string FileName)
1051 {
1052    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1053    string NewFileName;
1054    if (DirStrip.empty() == false &&
1055        FileName.length() > DirStrip.length() &&
1056        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1057                  DirStrip.begin(),DirStrip.end()) == 0)
1058    {
1059       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1060       while (NewFileName[0] == '/')
1061          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1062    }
1063    else
1064       NewFileName = FileName;
1065
1066    if (PathPrefix.empty() == false)
1067       NewFileName = flCombine(PathPrefix,NewFileName);
1068
1069    FileFd fd(FileName, FileFd::ReadOnly);
1070
1071    if (!fd.IsOpen())
1072    {
1073       return false;
1074    }
1075
1076    CheckSums[NewFileName].size = fd.Size();
1077
1078    Hashes hs;
1079    hs.AddFD(fd, 0, DoHashes);
1080    CheckSums[NewFileName].Hashes = hs.GetHashStringList();
1081    fd.Close();
1082
1083    return true;
1084 }
1085
1086                                                                         /*}}}*/
1087 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1088 // ---------------------------------------------------------------------
1089 static void printChecksumTypeRecord(FILE * const Output, char const * const Type, map<string, ReleaseWriter::CheckSum> const &CheckSums)
1090 {
1091       fprintf(Output, "%s:\n", Type);
1092       for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
1093           I != CheckSums.end(); ++I)
1094       {
1095          HashString const * const hs = I->second.Hashes.find(Type);
1096          if (hs == NULL)
1097             continue;
1098          fprintf(Output, " %s %16llu %s\n",
1099                  hs->HashValue().c_str(),
1100                  (*I).second.size,
1101                  (*I).first.c_str());
1102       }
1103 }
1104 void ReleaseWriter::Finish()
1105 {
1106    if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
1107       printChecksumTypeRecord(Output, "MD5Sum", CheckSums);
1108    if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
1109       printChecksumTypeRecord(Output, "SHA1", CheckSums);
1110    if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
1111       printChecksumTypeRecord(Output, "SHA256", CheckSums);
1112    if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
1113       printChecksumTypeRecord(Output, "SHA512", CheckSums);
1114 }