ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/error.h>
  18 #include <apt-pkg/configuration.h>
  19 #include <apt-pkg/aptconfiguration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/deblistparser.h>
  23
  24 #include <sys/types.h>
  25 #include <unistd.h>
  26 #include <ctime>
  27 #include <ftw.h>
  28 #include <fnmatch.h>
  29 #include <iostream>
  30 #include <sstream>
  31 #include <memory>
  32
  33 #include "writer.h"
  34 #include "cachedb.h"
  35 #include "apt-ftparchive.h"
  36 #include "multicompress.h"
  37
  38 #include <apti18n.h>
  39                                                                         /*}}}*/
  40 using namespace std;
  41 FTWScanner *FTWScanner::Owner;
  42
  43 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* */
  46 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  47                              const char *tag,
  48                              const char *rewrite,
  49                              const char *newtag = 0)
  50 {
  51     tfrd.Tag = tag;
  52     tfrd.Rewrite = rewrite;
  53     tfrd.NewTag = newtag;
  54 }
  55                                                                         /*}}}*/
  56
  57 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  58 // ---------------------------------------------------------------------
  59 /* */
  60 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  61 {
  62    ErrorPrinted = false;
  63    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  64
  65    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  66    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  67    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  68    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  69 }
  70                                                                         /*}}}*/
  71 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  72 // ---------------------------------------------------------------------
  73 /* This is the FTW scanner, it processes each directory element in the
  74    directory tree. */
  75 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  76 {
  77    if (Flag == FTW_DNR)
  78    {
  79       Owner->NewLine(1);
  80       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  81    }
  82    if (Flag == FTW_NS)
  83    {
  84       Owner->NewLine(1);
  85       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  86    }
  87    if (Flag != FTW_F)
  88       return 0;
  89
  90    return ScannerFile(File, true);
  91 }
  92                                                                         /*}}}*/
  93 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  94 // ---------------------------------------------------------------------
  95 /* */
  96 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  97 {
  98    const char *LastComponent = strrchr(File, '/');
  99    char *RealPath = NULL;
 100
 101    if (LastComponent == NULL)
 102       LastComponent = File;
 103    else
 104       LastComponent++;
 105
 106    vector<string>::const_iterator I;
 107    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 108    {
 109       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 110          break;
 111    }
 112    if (I == Owner->Patterns.end())
 113       return 0;
 114
 115    /* Process it. If the file is a link then resolve it into an absolute
 116       name.. This works best if the directory components the scanner are
 117       given are not links themselves. */
 118    char Jnk[2];
 119    Owner->OriginalPath = File;
 120    if (ReadLink &&
 121        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 122        (RealPath = realpath(File,NULL)) != 0)
 123    {
 124       Owner->DoPackage(RealPath);
 125       free(RealPath);
 126    }
 127    else
 128       Owner->DoPackage(File);
 129
 130    if (_error->empty() == false)
 131    {
 132       // Print any errors or warnings found
 133       string Err;
 134       bool SeenPath = false;
 135       while (_error->empty() == false)
 136       {
 137          Owner->NewLine(1);
 138
 139          bool const Type = _error->PopMessage(Err);
 140          if (Type == true)
 141             cerr << _("E: ") << Err << endl;
 142          else
 143             cerr << _("W: ") << Err << endl;
 144
 145          if (Err.find(File) != string::npos)
 146             SeenPath = true;
 147       }
 148
 149       if (SeenPath == false)
 150          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 151       return 0;
 152    }
 153
 154    return 0;
 155 }
 156                                                                         /*}}}*/
 157 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 158 // ---------------------------------------------------------------------
 159 /* */
 160 bool FTWScanner::RecursiveScan(string const &Dir)
 161 {
 162    char *RealPath = NULL;
 163    /* If noprefix is set then jam the scan root in, so we don't generate
 164       link followed paths out of control */
 165    if (InternalPrefix.empty() == true)
 166    {
 167       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 168          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 169       InternalPrefix = RealPath;
 170       free(RealPath);
 171    }
 172
 173    // Do recursive directory searching
 174    Owner = this;
 175    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 176
 177    // Error treewalking?
 178    if (Res != 0)
 179    {
 180       if (_error->PendingError() == false)
 181          _error->Errno("ftw",_("Tree walking failed"));
 182       return false;
 183    }
 184
 185    return true;
 186 }
 187                                                                         /*}}}*/
 188 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 189 // ---------------------------------------------------------------------
 190 /* This is an alternative to using FTW to locate files, it reads the list
 191    of files from another file. */
 192 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 193 {
 194    char *RealPath = NULL;
 195    /* If noprefix is set then jam the scan root in, so we don't generate
 196       link followed paths out of control */
 197    if (InternalPrefix.empty() == true)
 198    {
 199       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 200          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 201       InternalPrefix = RealPath;
 202       free(RealPath);
 203    }
 204
 205    Owner = this;
 206    FILE *List = fopen(File.c_str(),"r");
 207    if (List == 0)
 208       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 209
 210    /* We are a tad tricky here.. We prefix the buffer with the directory
 211       name, that way if we need a full path with just use line.. Sneaky and
 212       fully evil. */
 213    char Line[1000];
 214    char *FileStart;
 215    if (Dir.empty() == true || Dir.end()[-1] != '/')
 216       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 217    else
 218       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 219    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 220    {
 221       char *FileName = _strstrip(FileStart);
 222       if (FileName[0] == 0)
 223          continue;
 224
 225       if (FileName[0] != '/')
 226       {
 227          if (FileName != FileStart)
 228             memmove(FileStart,FileName,strlen(FileStart));
 229          FileName = Line;
 230       }
 231
 232 #if 0
 233       struct stat St;
 234       int Flag = FTW_F;
 235       if (stat(FileName,&St) != 0)
 236          Flag = FTW_NS;
 237 #endif
 238
 239       if (ScannerFile(FileName, false) != 0)
 240          break;
 241    }
 242
 243    fclose(List);
 244    return true;
 245 }
 246                                                                         /*}}}*/
 247 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* */
 250 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 251                         unsigned long long &DeLinkBytes,
 252                         unsigned long long const &FileSize)
 253 {
 254    // See if this isn't an internaly prefix'd file name.
 255    if (InternalPrefix.empty() == false &&
 256        InternalPrefix.length() < FileName.length() &&
 257        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 258                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 259    {
 260       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 261       {
 262          // Tidy up the display
 263          if (DeLinkBytes == 0)
 264             cout << endl;
 265
 266          NewLine(1);
 267          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 268                     SizeToStr(FileSize).c_str());
 269          c1out << flush;
 270
 271          if (NoLinkAct == false)
 272          {
 273             char OldLink[400];
 274             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 275                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 276             else
 277             {
 278                if (unlink(OriginalPath) != 0)
 279                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 280                else
 281                {
 282                   if (link(FileName.c_str(),OriginalPath) != 0)
 283                   {
 284                      // Panic! Restore the symlink
 285                      symlink(OldLink,OriginalPath);
 286                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 287                                           FileName.c_str(),
 288                                           OriginalPath);
 289                   }
 290                }
 291             }
 292          }
 293
 294          DeLinkBytes += FileSize;
 295          if (DeLinkBytes/1024 >= DeLinkLimit)
 296             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 297       }
 298
 299       FileName = OriginalPath;
 300    }
 301
 302    return true;
 303 }
 304                                                                         /*}}}*/
 305
 306 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 307 // ---------------------------------------------------------------------
 308 /* */
 309 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 310                                string const &Arch) :
 311    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 312 {
 313    Output = stdout;
 314    SetExts(".deb .udeb");
 315    DeLinkLimit = 0;
 316
 317    // Process the command line options
 318    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 319    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 320    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 321    DoSHA512 = _config->FindB("APT::FTPArchive::Packages::SHA512",DoSHA512);
 322    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 323    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 324    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 325    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 326
 327    if (Db.Loaded() == false)
 328       DoContents = false;
 329
 330    // Read the override file
 331    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 332       return;
 333    else
 334       NoOverride = true;
 335
 336    if (ExtOverrides.empty() == false)
 337       Over.ReadExtraOverride(ExtOverrides);
 338
 339    _error->DumpErrors();
 340 }
 341                                                                         /*}}}*/
 342 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 343 // ---------------------------------------------------------------------
 344 /* */
 345 bool FTWScanner::SetExts(string const &Vals)
 346 {
 347    ClearPatterns();
 348    string::size_type Start = 0;
 349    while (Start <= Vals.length()-1)
 350    {
 351       string::size_type const Space = Vals.find(' ',Start);
 352       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 353       if ( Arch.empty() == false )
 354       {
 355          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 356          AddPattern(string("*_all") + Vals.substr(Start, Length));
 357       }
 358       else
 359          AddPattern(string("*") + Vals.substr(Start, Length));
 360
 361       Start += Length + 1;
 362    }
 363
 364    return true;
 365 }
 366
 367                                                                         /*}}}*/
 368 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 369 // ---------------------------------------------------------------------
 370 /* This method takes a package and gets its control information and
 371    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 372    rewritten and the path/size/hash appended. */
 373 bool PackagesWriter::DoPackage(string FileName)
 374 {
 375    // Pull all the data we need form the DB
 376    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 377                   == false)
 378    {
 379       return false;
 380    }
 381
 382    unsigned long long FileSize = Db.GetFileSize();
 383    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 384       return false;
 385
 386    // Lookup the overide information
 387    pkgTagSection &Tags = Db.Control.Section;
 388    string Package = Tags.FindS("Package");
 389    string Architecture;
 390    // if we generate a Packages file for a given arch, we use it to
 391    // look for overrides. if we run in "simple" mode without the
 392    // "Architecures" variable in the config we use the architecure value
 393    // from the deb file
 394    if(Arch != "")
 395       Architecture = Arch;
 396    else
 397       Architecture = Tags.FindS("Architecture");
 398    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 399
 400    if (Package.empty() == true)
 401       return _error->Error(_("Archive had no package field"));
 402
 403    // If we need to do any rewriting of the header do it now..
 404    if (OverItem.get() == 0)
 405    {
 406       if (NoOverride == false)
 407       {
 408          NewLine(1);
 409          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 410       }
 411
 412       OverItem = auto_ptr<Override::Item>(new Override::Item);
 413       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 414       OverItem->Priority = Tags.FindS("Priority");
 415    }
 416
 417    char Size[40];
 418    sprintf(Size,"%llu", (unsigned long long) FileSize);
 419
 420    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 421    string NewFileName;
 422    if (DirStrip.empty() == false &&
 423        FileName.length() > DirStrip.length() &&
 424        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 425                  DirStrip.begin(),DirStrip.end()) == 0)
 426       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 427    else
 428       NewFileName = FileName;
 429    if (PathPrefix.empty() == false)
 430       NewFileName = flCombine(PathPrefix,NewFileName);
 431
 432    /* Configuration says we don't want to include the long Description
 433       in the package file - instead we want to ship a separated file */
 434    string desc;
 435    if (LongDescription == false) {
 436       desc = Tags.FindS("Description").append("\n");
 437       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 438    }
 439
 440    // This lists all the changes to the fields we are going to make.
 441    // (7 hardcoded + maintainer + suggests + end marker)
 442    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 443
 444    unsigned int End = 0;
 445    SetTFRewriteData(Changes[End++], "Size", Size);
 446    if (DoMD5 == true)
 447       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 448    if (DoSHA1 == true)
 449       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 450    if (DoSHA256 == true)
 451       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 452    if (DoSHA512 == true)
 453       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 454    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 455    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 456    SetTFRewriteData(Changes[End++], "Status", 0);
 457    SetTFRewriteData(Changes[End++], "Optional", 0);
 458
 459    string DescriptionMd5;
 460    if (LongDescription == false) {
 461       MD5Summation descmd5;
 462       descmd5.Add(desc.c_str());
 463       DescriptionMd5 = descmd5.Result().Value();
 464       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 465       if (TransWriter != NULL)
 466          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 467    }
 468
 469    // Rewrite the maintainer field if necessary
 470    bool MaintFailed;
 471    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 472    if (MaintFailed == true)
 473    {
 474       if (NoOverride == false)
 475       {
 476          NewLine(1);
 477          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 478                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 479       }
 480    }
 481
 482    if (NewMaint.empty() == false)
 483       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 484
 485    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 486       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 487       but dpkg does this append bit. So we do the append bit, at least that way the
 488       status file and package file will remain similar. There are other transforms
 489       but optional is the only legacy one still in use for some lazy reason. */
 490    string OptionalStr = Tags.FindS("Optional");
 491    if (OptionalStr.empty() == false)
 492    {
 493       if (Tags.FindS("Suggests").empty() == false)
 494          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 495       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 496    }
 497
 498    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 499         I != OverItem->FieldOverride.end(); ++I)
 500       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 501
 502    SetTFRewriteData(Changes[End++], 0, 0);
 503
 504    // Rewrite and store the fields.
 505    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 506       return false;
 507    fprintf(Output,"\n");
 508
 509    return Db.Finish();
 510 }
 511                                                                         /*}}}*/
 512
 513 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 514 // ---------------------------------------------------------------------
 515 /* Create a Translation-Master file for this Packages file */
 516 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 517                                         mode_t const &Permissions) : Output(NULL),
 518                                                         RefCounter(0)
 519 {
 520    if (File.empty() == true)
 521       return;
 522
 523    Comp = new MultiCompress(File, TransCompress, Permissions);
 524    Output = Comp->Input;
 525 }
 526                                                                         /*}}}*/
 527 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 528 // ---------------------------------------------------------------------
 529 /* Create a Translation-Master file for this Packages file */
 530 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 531                                   string const &MD5)
 532 {
 533    if (Output == NULL)
 534       return true;
 535
 536    // Different archs can include different versions and therefore
 537    // different descriptions - so we need to check for both name and md5.
 538    string const Record = Pkg + ":" + MD5;
 539
 540    if (Included.find(Record) != Included.end())
 541       return true;
 542
 543    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 544            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 545
 546    Included.insert(Record);
 547    return true;
 548 }
 549                                                                         /*}}}*/
 550 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 551 // ---------------------------------------------------------------------
 552 /* */
 553 TranslationWriter::~TranslationWriter()
 554 {
 555    if (Comp == NULL)
 556       return;
 557
 558    delete Comp;
 559 }
 560                                                                         /*}}}*/
 561
 562 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 563 // ---------------------------------------------------------------------
 564 /* */
 565 SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
 566                              string const &ExtOverrides) :
 567    Db(DB), Stats(Db.Stats)
 568 {
 569    Output = stdout;
 570    AddPattern("*.dsc");
 571    DeLinkLimit = 0;
 572    Buffer = 0;
 573    BufSize = 0;
 574
 575    // Process the command line options
 576    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 577    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 578    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 579    DoSHA512 = _config->FindB("APT::FTPArchive::Sources::SHA512",DoSHA512);
 580    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 581    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 582
 583    // Read the override file
 584    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 585       return;
 586    else
 587       NoOverride = true;
 588
 589    // WTF?? The logic above: if we can't read binary overrides, don't even try
 590    // reading source overrides. if we can read binary overrides, then say there
 591    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 592
 593    if (ExtOverrides.empty() == false)
 594       SOver.ReadExtraOverride(ExtOverrides);
 595
 596    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 597       SOver.ReadOverride(SOverrides,true);
 598 }
 599                                                                         /*}}}*/
 600 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* */
 603 bool SourcesWriter::DoPackage(string FileName)
 604 {
 605    // Open the archive
 606    FileFd F(FileName,FileFd::ReadOnly);
 607    if (_error->PendingError() == true)
 608       return false;
 609
 610    // Stat the file for later
 611    struct stat St;
 612    if (fstat(F.Fd(),&St) != 0)
 613       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 614
 615    if (St.st_size > 128*1024)
 616       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 617
 618    if (BufSize < (unsigned long long)St.st_size+1)
 619    {
 620       BufSize = St.st_size+1;
 621       Buffer = (char *)realloc(Buffer,St.st_size+1);
 622    }
 623
 624    if (F.Read(Buffer,St.st_size) == false)
 625       return false;
 626
 627    // Hash the file
 628    char *Start = Buffer;
 629    char *BlkEnd = Buffer + St.st_size;
 630
 631    MD5Summation MD5;
 632    SHA1Summation SHA1;
 633    SHA256Summation SHA256;
 634    SHA256Summation SHA512;
 635
 636    if (DoMD5 == true)
 637       MD5.Add((unsigned char *)Start,BlkEnd - Start);
 638    if (DoSHA1 == true)
 639       SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 640    if (DoSHA256 == true)
 641       SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 642    if (DoSHA512 == true)
 643       SHA512.Add((unsigned char *)Start,BlkEnd - Start);
 644
 645    // Add an extra \n to the end, just in case
 646    *BlkEnd++ = '\n';
 647
 648    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 649       before */
 650    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 651    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 652    {
 653       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 654       {
 655          MsgEnd[1] = '\n';
 656          break;
 657       }
 658    }
 659
 660    /* Read records until we locate the Source record. This neatly skips the
 661       GPG header (which is RFC822 formed) without any trouble. */
 662    pkgTagSection Tags;
 663    do
 664    {
 665       unsigned Pos;
 666       if (Tags.Scan(Start,BlkEnd - Start) == false)
 667          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 668       if (Tags.Find("Source",Pos) == true)
 669          break;
 670       Start += Tags.size();
 671    }
 672    while (1);
 673    Tags.Trim();
 674
 675    // Lookup the overide information, finding first the best priority.
 676    string BestPrio;
 677    string Bins = Tags.FindS("Binary");
 678    char Buffer[Bins.length() + 1];
 679    auto_ptr<Override::Item> OverItem(0);
 680    if (Bins.empty() == false)
 681    {
 682       strcpy(Buffer,Bins.c_str());
 683
 684       // Ignore too-long errors.
 685       char *BinList[400];
 686       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 687
 688       // Look at all the binaries
 689       unsigned char BestPrioV = pkgCache::State::Extra;
 690       for (unsigned I = 0; BinList[I] != 0; I++)
 691       {
 692          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 693          if (Itm.get() == 0)
 694             continue;
 695
 696          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 697          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 698          {
 699             BestPrioV = NewPrioV;
 700             BestPrio = Itm->Priority;
 701          }
 702
 703          if (OverItem.get() == 0)
 704             OverItem = Itm;
 705       }
 706    }
 707
 708    // If we need to do any rewriting of the header do it now..
 709    if (OverItem.get() == 0)
 710    {
 711       if (NoOverride == false)
 712       {
 713          NewLine(1);
 714          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 715       }
 716
 717       OverItem = auto_ptr<Override::Item>(new Override::Item);
 718    }
 719
 720    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 721    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 722    if (SOverItem.get() == 0)
 723    {
 724       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 725       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 726       if (SOverItem.get() == 0)
 727       {
 728         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 729          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 730          *SOverItem = *OverItem;
 731       }
 732    }
 733
 734    // Add the dsc to the files hash list
 735    string const strippedName = flNotDir(FileName);
 736    std::ostringstream ostreamFiles;
 737    if (DoMD5 == true && Tags.Exists("Files"))
 738       ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 739                    << strippedName << "\n " << Tags.FindS("Files");
 740    string const Files = ostreamFiles.str();
 741
 742    std::ostringstream ostreamSha1;
 743    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 744       ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 745                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 746
 747    std::ostringstream ostreamSha256;
 748    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 749       ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 750                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 751
 752    std::ostringstream ostreamSha512;
 753    if (DoSHA512 == true && Tags.Exists("Checksums-Sha512"))
 754       ostreamSha512 << "\n " << string(SHA512.Result()) << " " << St.st_size << " "
 755                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 756
 757    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 758    string NewFileName;
 759    if (DirStrip.empty() == false &&
 760        FileName.length() > DirStrip.length() &&
 761        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 762       NewFileName = string(OriginalPath + DirStrip.length());
 763    else
 764       NewFileName = OriginalPath;
 765    if (PathPrefix.empty() == false)
 766       NewFileName = flCombine(PathPrefix,NewFileName);
 767
 768    string Directory = flNotFile(OriginalPath);
 769    string Package = Tags.FindS("Source");
 770
 771    // Perform operation over all of the files
 772    string ParseJnk;
 773    const char *C = Files.c_str();
 774    char *RealPath = NULL;
 775    for (;isspace(*C); C++);
 776    while (*C != 0)
 777    {
 778       // Parse each of the elements
 779       if (ParseQuoteWord(C,ParseJnk) == false ||
 780           ParseQuoteWord(C,ParseJnk) == false ||
 781           ParseQuoteWord(C,ParseJnk) == false)
 782          return _error->Error("Error parsing file record");
 783
 784       string OriginalPath = Directory + ParseJnk;
 785
 786       // Add missing hashes to source files
 787       if ((DoSHA1 == true && !Tags.Exists("Checksums-Sha1")) ||
 788           (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) ||
 789           (DoSHA512 == true && !Tags.Exists("Checksums-Sha512")))
 790       {
 791          if (Db.GetFileInfo(OriginalPath, false, false, false, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 792                == false)
 793          {
 794             return _error->Error("Error getting file info");
 795          }
 796
 797          if (DoSHA1 == true && !Tags.Exists("Checksums-Sha1"))
 798             ostreamSha1 << "\n " << string(Db.SHA1Res) << " "
 799                << Db.GetFileSize() << " " << ParseJnk;
 800
 801          if (DoSHA256 == true && !Tags.Exists("Checksums-Sha256"))
 802             ostreamSha256 << "\n " << string(Db.SHA256Res) << " "
 803                << Db.GetFileSize() << " " << ParseJnk;
 804
 805          if (DoSHA512 == true && !Tags.Exists("Checksums-Sha512"))
 806             ostreamSha512 << "\n " << string(Db.SHA512Res) << " "
 807                << Db.GetFileSize() << " " << ParseJnk;
 808       }
 809
 810       // Perform the delinking operation
 811       char Jnk[2];
 812
 813       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 814           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 815       {
 816          string RP = RealPath;
 817          free(RealPath);
 818          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 819             return false;
 820       }
 821    }
 822
 823    Directory = flNotFile(NewFileName);
 824    if (Directory.length() > 2)
 825       Directory.erase(Directory.end()-1);
 826
 827    string const ChecksumsSha1 = ostreamSha1.str();
 828    string const ChecksumsSha256 = ostreamSha256.str();
 829    string const ChecksumsSha512 = ostreamSha512.str();
 830
 831    // This lists all the changes to the fields we are going to make.
 832    // (5 hardcoded + checksums + maintainer + end marker)
 833    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 834
 835    unsigned int End = 0;
 836    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 837    if (Files.empty() == false)
 838       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 839    if (ChecksumsSha1.empty() == false)
 840       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 841    if (ChecksumsSha256.empty() == false)
 842       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 843    if (ChecksumsSha512.empty() == false)
 844       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 845    if (Directory != "./")
 846       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 847    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 848    SetTFRewriteData(Changes[End++],"Status",0);
 849
 850    // Rewrite the maintainer field if necessary
 851    bool MaintFailed;
 852    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 853    if (MaintFailed == true)
 854    {
 855       if (NoOverride == false)
 856       {
 857          NewLine(1);
 858          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 859                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 860       }
 861    }
 862    if (NewMaint.empty() == false)
 863       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 864
 865    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 866         I != SOverItem->FieldOverride.end(); ++I)
 867       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 868
 869    SetTFRewriteData(Changes[End++], 0, 0);
 870
 871    // Rewrite and store the fields.
 872    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 873       return false;
 874    fprintf(Output,"\n");
 875
 876    Stats.Packages++;
 877
 878    return Db.Finish();
 879 }
 880                                                                         /*}}}*/
 881
 882 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 883 // ---------------------------------------------------------------------
 884 /* */
 885 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 886                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 887
 888 {
 889    SetExts(".deb");
 890    Output = stdout;
 891 }
 892                                                                         /*}}}*/
 893 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 894 // ---------------------------------------------------------------------
 895 /* If Package is the empty string the control record will be parsed to
 896    determine what the package name is. */
 897 bool ContentsWriter::DoPackage(string FileName, string Package)
 898 {
 899    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 900    {
 901       return false;
 902    }
 903
 904    // Parse the package name
 905    if (Package.empty() == true)
 906    {
 907       Package = Db.Control.Section.FindS("Package");
 908    }
 909
 910    Db.Contents.Add(Gen,Package);
 911
 912    return Db.Finish();
 913 }
 914                                                                         /*}}}*/
 915 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* */
 918 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 919 {
 920    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 921    if (_error->PendingError() == true)
 922       return false;
 923
 924    // Open the package file
 925    FileFd Fd;
 926    if (Pkgs.OpenOld(Fd) == false)
 927       return false;
 928
 929    pkgTagFile Tags(&Fd);
 930    if (_error->PendingError() == true)
 931       return false;
 932
 933    // Parse.
 934    pkgTagSection Section;
 935    while (Tags.Step(Section) == true)
 936    {
 937       string File = flCombine(Prefix,Section.FindS("FileName"));
 938       string Package = Section.FindS("Section");
 939       if (Package.empty() == false && Package.end()[-1] != '/')
 940       {
 941          Package += '/';
 942          Package += Section.FindS("Package");
 943       }
 944       else
 945          Package += Section.FindS("Package");
 946
 947       DoPackage(File,Package);
 948       if (_error->empty() == false)
 949       {
 950          _error->Error("Errors apply to file '%s'",File.c_str());
 951          _error->DumpErrors();
 952       }
 953    }
 954
 955    // Tidy the compressor
 956    Fd.Close();
 957
 958    return true;
 959 }
 960
 961                                                                         /*}}}*/
 962
 963 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 964 // ---------------------------------------------------------------------
 965 /* */
 966 ReleaseWriter::ReleaseWriter(string const &DB)
 967 {
 968    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 969    {
 970       AddPattern("Packages");
 971       AddPattern("Packages.gz");
 972       AddPattern("Packages.bz2");
 973       AddPattern("Packages.lzma");
 974       AddPattern("Packages.xz");
 975       AddPattern("Translation-*");
 976       AddPattern("Sources");
 977       AddPattern("Sources.gz");
 978       AddPattern("Sources.bz2");
 979       AddPattern("Sources.lzma");
 980       AddPattern("Sources.xz");
 981       AddPattern("Release");
 982       AddPattern("Contents-*");
 983       AddPattern("Index");
 984       AddPattern("md5sum.txt");
 985    }
 986    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 987
 988    Output = stdout;
 989    time_t const now = time(NULL);
 990
 991    setlocale(LC_TIME, "C");
 992
 993    char datestr[128];
 994    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 995                 gmtime(&now)) == 0)
 996    {
 997       datestr[0] = '\0';
 998    }
 999
1000    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
1001    char validstr[128];
1002    if (now == validuntil ||
1003        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
1004                 gmtime(&validuntil)) == 0)
1005    {
1006       validstr[0] = '\0';
1007    }
1008
1009    setlocale(LC_TIME, "");
1010
1011    map<string,string> Fields;
1012    Fields["Origin"] = "";
1013    Fields["Label"] = "";
1014    Fields["Suite"] = "";
1015    Fields["Version"] = "";
1016    Fields["Codename"] = "";
1017    Fields["Date"] = datestr;
1018    Fields["Valid-Until"] = validstr;
1019    Fields["Architectures"] = "";
1020    Fields["Components"] = "";
1021    Fields["Description"] = "";
1022
1023    for(map<string,string>::const_iterator I = Fields.begin();
1024        I != Fields.end();
1025        ++I)
1026    {
1027       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1028       string Value = _config->Find(Config, (*I).second.c_str());
1029       if (Value == "")
1030          continue;
1031
1032       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1033    }
1034
1035    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1036    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1037    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1038 }
1039                                                                         /*}}}*/
1040 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1041 // ---------------------------------------------------------------------
1042 bool ReleaseWriter::DoPackage(string FileName)
1043 {
1044    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1045    string NewFileName;
1046    if (DirStrip.empty() == false &&
1047        FileName.length() > DirStrip.length() &&
1048        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1049                  DirStrip.begin(),DirStrip.end()) == 0)
1050    {
1051       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1052       while (NewFileName[0] == '/')
1053          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1054    }
1055    else
1056       NewFileName = FileName;
1057
1058    if (PathPrefix.empty() == false)
1059       NewFileName = flCombine(PathPrefix,NewFileName);
1060
1061    FileFd fd(FileName, FileFd::ReadOnly);
1062
1063    if (!fd.IsOpen())
1064    {
1065       return false;
1066    }
1067
1068    CheckSums[NewFileName].size = fd.Size();
1069
1070    Hashes hs;
1071    hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1072    if (DoMD5 == true)
1073       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1074    if (DoSHA1 == true)
1075       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1076    if (DoSHA256 == true)
1077       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1078    if (DoSHA512 == true)
1079       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1080    fd.Close();
1081
1082    return true;
1083 }
1084
1085                                                                         /*}}}*/
1086 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1087 // ---------------------------------------------------------------------
1088 void ReleaseWriter::Finish()
1089 {
1090    if (DoMD5 == true)
1091    {
1092       fprintf(Output, "MD5Sum:\n");
1093       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1094           I != CheckSums.end(); ++I)
1095       {
1096          fprintf(Output, " %s %16llu %s\n",
1097                  (*I).second.MD5.c_str(),
1098                  (*I).second.size,
1099                  (*I).first.c_str());
1100       }
1101    }
1102    if (DoSHA1 == true)
1103    {
1104       fprintf(Output, "SHA1:\n");
1105       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1106           I != CheckSums.end(); ++I)
1107       {
1108          fprintf(Output, " %s %16llu %s\n",
1109                  (*I).second.SHA1.c_str(),
1110                  (*I).second.size,
1111                  (*I).first.c_str());
1112       }
1113    }
1114    if (DoSHA256 == true)
1115    {
1116       fprintf(Output, "SHA256:\n");
1117       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1118           I != CheckSums.end(); ++I)
1119       {
1120          fprintf(Output, " %s %16llu %s\n",
1121                  (*I).second.SHA256.c_str(),
1122                  (*I).second.size,
1123                  (*I).first.c_str());
1124       }
1125    }
1126
1127    fprintf(Output, "SHA512:\n");
1128    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1129        I != CheckSums.end();
1130        ++I)
1131    {
1132       fprintf(Output, " %s %16llu %s\n",
1133               (*I).second.SHA512.c_str(),
1134               (*I).second.size,
1135               (*I).first.c_str());
1136    }
1137
1138 }