ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include "writer.h"
  15
  16 #include <apti18n.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/md5.h>
  22 #include <apt-pkg/hashes.h>
  23 #include <apt-pkg/deblistparser.h>
  24
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <ctime>
  28 #include <ftw.h>
  29 #include <fnmatch.h>
  30 #include <iostream>
  31 #include <sstream>
  32 #include <memory>
  33
  34 #include "cachedb.h"
  35 #include "apt-ftparchive.h"
  36 #include "multicompress.h"
  37                                                                         /*}}}*/
  38 using namespace std;
  39 FTWScanner *FTWScanner::Owner;
  40
  41 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  45                              const char *tag,
  46                              const char *rewrite,
  47                              const char *newtag = 0)
  48 {
  49     tfrd.Tag = tag;
  50     tfrd.Rewrite = rewrite;
  51     tfrd.NewTag = newtag;
  52 }
  53                                                                         /*}}}*/
  54
  55 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  59 {
  60    ErrorPrinted = false;
  61    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  62
  63    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  64    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  65    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  66    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  67 }
  68                                                                         /*}}}*/
  69 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  70 // ---------------------------------------------------------------------
  71 /* This is the FTW scanner, it processes each directory element in the
  72    directory tree. */
  73 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  74 {
  75    if (Flag == FTW_DNR)
  76    {
  77       Owner->NewLine(1);
  78       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  79    }
  80    if (Flag == FTW_NS)
  81    {
  82       Owner->NewLine(1);
  83       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  84    }
  85    if (Flag != FTW_F)
  86       return 0;
  87
  88    return ScannerFile(File, true);
  89 }
  90                                                                         /*}}}*/
  91 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  92 // ---------------------------------------------------------------------
  93 /* */
  94 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  95 {
  96    const char *LastComponent = strrchr(File, '/');
  97    char *RealPath = NULL;
  98
  99    if (LastComponent == NULL)
 100       LastComponent = File;
 101    else
 102       LastComponent++;
 103
 104    vector<string>::const_iterator I;
 105    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 106    {
 107       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 108          break;
 109    }
 110    if (I == Owner->Patterns.end())
 111       return 0;
 112
 113    /* Process it. If the file is a link then resolve it into an absolute
 114       name.. This works best if the directory components the scanner are
 115       given are not links themselves. */
 116    char Jnk[2];
 117    Owner->OriginalPath = File;
 118    if (ReadLink &&
 119        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 120        (RealPath = realpath(File,NULL)) != 0)
 121    {
 122       Owner->DoPackage(RealPath);
 123       free(RealPath);
 124    }
 125    else
 126       Owner->DoPackage(File);
 127
 128    if (_error->empty() == false)
 129    {
 130       // Print any errors or warnings found
 131       string Err;
 132       bool SeenPath = false;
 133       while (_error->empty() == false)
 134       {
 135          Owner->NewLine(1);
 136
 137          bool const Type = _error->PopMessage(Err);
 138          if (Type == true)
 139             cerr << _("E: ") << Err << endl;
 140          else
 141             cerr << _("W: ") << Err << endl;
 142
 143          if (Err.find(File) != string::npos)
 144             SeenPath = true;
 145       }
 146
 147       if (SeenPath == false)
 148          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 149       return 0;
 150    }
 151
 152    return 0;
 153 }
 154                                                                         /*}}}*/
 155 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* */
 158 bool FTWScanner::RecursiveScan(string const &Dir)
 159 {
 160    char *RealPath = NULL;
 161    /* If noprefix is set then jam the scan root in, so we don't generate
 162       link followed paths out of control */
 163    if (InternalPrefix.empty() == true)
 164    {
 165       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 166          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 167       InternalPrefix = RealPath;
 168       free(RealPath);
 169    }
 170
 171    // Do recursive directory searching
 172    Owner = this;
 173    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 174
 175    // Error treewalking?
 176    if (Res != 0)
 177    {
 178       if (_error->PendingError() == false)
 179          _error->Errno("ftw",_("Tree walking failed"));
 180       return false;
 181    }
 182
 183    return true;
 184 }
 185                                                                         /*}}}*/
 186 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 187 // ---------------------------------------------------------------------
 188 /* This is an alternative to using FTW to locate files, it reads the list
 189    of files from another file. */
 190 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 191 {
 192    char *RealPath = NULL;
 193    /* If noprefix is set then jam the scan root in, so we don't generate
 194       link followed paths out of control */
 195    if (InternalPrefix.empty() == true)
 196    {
 197       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 198          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 199       InternalPrefix = RealPath;
 200       free(RealPath);
 201    }
 202
 203    Owner = this;
 204    FILE *List = fopen(File.c_str(),"r");
 205    if (List == 0)
 206       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 207
 208    /* We are a tad tricky here.. We prefix the buffer with the directory
 209       name, that way if we need a full path with just use line.. Sneaky and
 210       fully evil. */
 211    char Line[1000];
 212    char *FileStart;
 213    if (Dir.empty() == true || Dir.end()[-1] != '/')
 214       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 215    else
 216       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 217    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 218    {
 219       char *FileName = _strstrip(FileStart);
 220       if (FileName[0] == 0)
 221          continue;
 222
 223       if (FileName[0] != '/')
 224       {
 225          if (FileName != FileStart)
 226             memmove(FileStart,FileName,strlen(FileStart));
 227          FileName = Line;
 228       }
 229
 230 #if 0
 231       struct stat St;
 232       int Flag = FTW_F;
 233       if (stat(FileName,&St) != 0)
 234          Flag = FTW_NS;
 235 #endif
 236
 237       if (ScannerFile(FileName, false) != 0)
 238          break;
 239    }
 240
 241    fclose(List);
 242    return true;
 243 }
 244                                                                         /*}}}*/
 245 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 246 // ---------------------------------------------------------------------
 247 /* */
 248 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 249                         unsigned long &DeLinkBytes,
 250                         off_t const &FileSize)
 251 {
 252    // See if this isn't an internaly prefix'd file name.
 253    if (InternalPrefix.empty() == false &&
 254        InternalPrefix.length() < FileName.length() &&
 255        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 256                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 257    {
 258       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 259       {
 260          // Tidy up the display
 261          if (DeLinkBytes == 0)
 262             cout << endl;
 263
 264          NewLine(1);
 265          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 266                     SizeToStr(FileSize).c_str());
 267          c1out << flush;
 268
 269          if (NoLinkAct == false)
 270          {
 271             char OldLink[400];
 272             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 273                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 274             else
 275             {
 276                if (unlink(OriginalPath) != 0)
 277                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 278                else
 279                {
 280                   if (link(FileName.c_str(),OriginalPath) != 0)
 281                   {
 282                      // Panic! Restore the symlink
 283                      symlink(OldLink,OriginalPath);
 284                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 285                                           FileName.c_str(),
 286                                           OriginalPath);
 287                   }
 288                }
 289             }
 290          }
 291
 292          DeLinkBytes += FileSize;
 293          if (DeLinkBytes/1024 >= DeLinkLimit)
 294             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 295       }
 296
 297       FileName = OriginalPath;
 298    }
 299
 300    return true;
 301 }
 302                                                                         /*}}}*/
 303
 304 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 305 // ---------------------------------------------------------------------
 306 /* */
 307 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 308                                string const &Arch) :
 309    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 310 {
 311    Output = stdout;
 312    SetExts(".deb .udeb");
 313    DeLinkLimit = 0;
 314
 315    // Process the command line options
 316    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 317    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 318    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 319    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA512",true);
 320    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 321    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 322    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 323    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 324
 325    if (Db.Loaded() == false)
 326       DoContents = false;
 327
 328    // Read the override file
 329    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 330       return;
 331    else
 332       NoOverride = true;
 333
 334    if (ExtOverrides.empty() == false)
 335       Over.ReadExtraOverride(ExtOverrides);
 336
 337    _error->DumpErrors();
 338 }
 339                                                                         /*}}}*/
 340 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* */
 343 bool FTWScanner::SetExts(string const &Vals)
 344 {
 345    ClearPatterns();
 346    string::size_type Start = 0;
 347    while (Start <= Vals.length()-1)
 348    {
 349       string::size_type const Space = Vals.find(' ',Start);
 350       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 351       if ( Arch.empty() == false )
 352       {
 353          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 354          AddPattern(string("*_all") + Vals.substr(Start, Length));
 355       }
 356       else
 357          AddPattern(string("*") + Vals.substr(Start, Length));
 358
 359       Start += Length + 1;
 360    }
 361
 362    return true;
 363 }
 364
 365                                                                         /*}}}*/
 366 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 367 // ---------------------------------------------------------------------
 368 /* This method takes a package and gets its control information and
 369    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 370    rewritten and the path/size/hash appended. */
 371 bool PackagesWriter::DoPackage(string FileName)
 372 {
 373    // Pull all the data we need form the DB
 374    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 375                   == false)
 376    {
 377       return false;
 378    }
 379
 380    off_t FileSize = Db.GetFileSize();
 381    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 382       return false;
 383
 384    // Lookup the overide information
 385    pkgTagSection &Tags = Db.Control.Section;
 386    string Package = Tags.FindS("Package");
 387    string Architecture;
 388    // if we generate a Packages file for a given arch, we use it to
 389    // look for overrides. if we run in "simple" mode without the
 390    // "Architecures" variable in the config we use the architecure value
 391    // from the deb file
 392    if(Arch != "")
 393       Architecture = Arch;
 394    else
 395       Architecture = Tags.FindS("Architecture");
 396    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 397
 398    if (Package.empty() == true)
 399       return _error->Error(_("Archive had no package field"));
 400
 401    // If we need to do any rewriting of the header do it now..
 402    if (OverItem.get() == 0)
 403    {
 404       if (NoOverride == false)
 405       {
 406          NewLine(1);
 407          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 408       }
 409
 410       OverItem = auto_ptr<Override::Item>(new Override::Item);
 411       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 412       OverItem->Priority = Tags.FindS("Priority");
 413    }
 414
 415    char Size[40];
 416    sprintf(Size,"%lu", (unsigned long) FileSize);
 417
 418    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 419    string NewFileName;
 420    if (DirStrip.empty() == false &&
 421        FileName.length() > DirStrip.length() &&
 422        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 423                  DirStrip.begin(),DirStrip.end()) == 0)
 424       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 425    else
 426       NewFileName = FileName;
 427    if (PathPrefix.empty() == false)
 428       NewFileName = flCombine(PathPrefix,NewFileName);
 429
 430    /* Configuration says we don't want to include the long Description
 431       in the package file - instead we want to ship a separated file */
 432    string desc;
 433    if (LongDescription == false) {
 434       desc = Tags.FindS("Description").append("\n");
 435       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 436    }
 437
 438    // This lists all the changes to the fields we are going to make.
 439    // (7 hardcoded + maintainer + suggests + end marker)
 440    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 441
 442    unsigned int End = 0;
 443    SetTFRewriteData(Changes[End++], "Size", Size);
 444    if (DoMD5 == true)
 445       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 446    if (DoSHA1 == true)
 447       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 448    if (DoSHA256 == true)
 449       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 450    if (DoSHA512 == true)
 451       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 452    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 453    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 454    SetTFRewriteData(Changes[End++], "Status", 0);
 455    SetTFRewriteData(Changes[End++], "Optional", 0);
 456
 457    string DescriptionMd5;
 458    if (LongDescription == false) {
 459       MD5Summation descmd5;
 460       descmd5.Add(desc.c_str());
 461       DescriptionMd5 = descmd5.Result().Value();
 462       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 463       if (TransWriter != NULL)
 464          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 465    }
 466
 467    // Rewrite the maintainer field if necessary
 468    bool MaintFailed;
 469    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 470    if (MaintFailed == true)
 471    {
 472       if (NoOverride == false)
 473       {
 474          NewLine(1);
 475          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 476                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 477       }
 478    }
 479
 480    if (NewMaint.empty() == false)
 481       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 482
 483    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 484       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 485       but dpkg does this append bit. So we do the append bit, at least that way the
 486       status file and package file will remain similar. There are other transforms
 487       but optional is the only legacy one still in use for some lazy reason. */
 488    string OptionalStr = Tags.FindS("Optional");
 489    if (OptionalStr.empty() == false)
 490    {
 491       if (Tags.FindS("Suggests").empty() == false)
 492          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 493       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 494    }
 495
 496    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 497         I != OverItem->FieldOverride.end(); ++I)
 498       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 499
 500    SetTFRewriteData(Changes[End++], 0, 0);
 501
 502    // Rewrite and store the fields.
 503    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 504       return false;
 505    fprintf(Output,"\n");
 506
 507    return Db.Finish();
 508 }
 509                                                                         /*}}}*/
 510
 511 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 512 // ---------------------------------------------------------------------
 513 /* Create a Translation-Master file for this Packages file */
 514 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 515                                         mode_t const &Permissions) : Output(NULL),
 516                                                         RefCounter(0)
 517 {
 518    if (File.empty() == true)
 519       return;
 520
 521    Comp = new MultiCompress(File, TransCompress, Permissions);
 522    Output = Comp->Input;
 523 }
 524                                                                         /*}}}*/
 525 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 526 // ---------------------------------------------------------------------
 527 /* Create a Translation-Master file for this Packages file */
 528 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 529                                   string const &MD5)
 530 {
 531    if (Output == NULL)
 532       return true;
 533
 534    // Different archs can include different versions and therefore
 535    // different descriptions - so we need to check for both name and md5.
 536    string const Record = Pkg + ":" + MD5;
 537
 538    if (Included.find(Record) != Included.end())
 539       return true;
 540
 541    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 542            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 543
 544    Included.insert(Record);
 545    return true;
 546 }
 547                                                                         /*}}}*/
 548 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* */
 551 TranslationWriter::~TranslationWriter()
 552 {
 553    if (Comp == NULL)
 554       return;
 555
 556    delete Comp;
 557 }
 558                                                                         /*}}}*/
 559
 560 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 561 // ---------------------------------------------------------------------
 562 /* */
 563 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 564                              string const &ExtOverrides)
 565 {
 566    Output = stdout;
 567    AddPattern("*.dsc");
 568    DeLinkLimit = 0;
 569    Buffer = 0;
 570    BufSize = 0;
 571
 572    // Process the command line options
 573    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 574    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 575    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 576    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 577
 578    // Read the override file
 579    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 580       return;
 581    else
 582       NoOverride = true;
 583
 584    // WTF?? The logic above: if we can't read binary overrides, don't even try
 585    // reading source overrides. if we can read binary overrides, then say there
 586    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 587
 588    if (ExtOverrides.empty() == false)
 589       SOver.ReadExtraOverride(ExtOverrides);
 590
 591    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 592       SOver.ReadOverride(SOverrides,true);
 593 }
 594                                                                         /*}}}*/
 595 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 596 // ---------------------------------------------------------------------
 597 /* */
 598 bool SourcesWriter::DoPackage(string FileName)
 599 {
 600    // Open the archive
 601    FileFd F(FileName,FileFd::ReadOnly);
 602    if (_error->PendingError() == true)
 603       return false;
 604
 605    // Stat the file for later
 606    struct stat St;
 607    if (fstat(F.Fd(),&St) != 0)
 608       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 609
 610    if (St.st_size > 128*1024)
 611       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 612
 613    if (BufSize < (unsigned)St.st_size+1)
 614    {
 615       BufSize = St.st_size+1;
 616       Buffer = (char *)realloc(Buffer,St.st_size+1);
 617    }
 618
 619    if (F.Read(Buffer,St.st_size) == false)
 620       return false;
 621
 622    // Hash the file
 623    char *Start = Buffer;
 624    char *BlkEnd = Buffer + St.st_size;
 625
 626    MD5Summation MD5;
 627    SHA1Summation SHA1;
 628    SHA256Summation SHA256;
 629    SHA256Summation SHA512;
 630
 631    if (DoMD5 == true)
 632       MD5.Add((unsigned char *)Start,BlkEnd - Start);
 633    if (DoSHA1 == true)
 634       SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 635    if (DoSHA256 == true)
 636       SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 637    if (DoSHA512 == true)
 638       SHA512.Add((unsigned char *)Start,BlkEnd - Start);
 639
 640    // Add an extra \n to the end, just in case
 641    *BlkEnd++ = '\n';
 642
 643    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 644       before */
 645    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 646    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 647    {
 648       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 649       {
 650          MsgEnd[1] = '\n';
 651          break;
 652       }
 653    }
 654
 655    /* Read records until we locate the Source record. This neatly skips the
 656       GPG header (which is RFC822 formed) without any trouble. */
 657    pkgTagSection Tags;
 658    do
 659    {
 660       unsigned Pos;
 661       if (Tags.Scan(Start,BlkEnd - Start) == false)
 662          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 663       if (Tags.Find("Source",Pos) == true)
 664          break;
 665       Start += Tags.size();
 666    }
 667    while (1);
 668    Tags.Trim();
 669
 670    // Lookup the overide information, finding first the best priority.
 671    string BestPrio;
 672    string Bins = Tags.FindS("Binary");
 673    char Buffer[Bins.length() + 1];
 674    auto_ptr<Override::Item> OverItem(0);
 675    if (Bins.empty() == false)
 676    {
 677       strcpy(Buffer,Bins.c_str());
 678
 679       // Ignore too-long errors.
 680       char *BinList[400];
 681       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 682
 683       // Look at all the binaries
 684       unsigned char BestPrioV = pkgCache::State::Extra;
 685       for (unsigned I = 0; BinList[I] != 0; I++)
 686       {
 687          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 688          if (Itm.get() == 0)
 689             continue;
 690
 691          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 692          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 693          {
 694             BestPrioV = NewPrioV;
 695             BestPrio = Itm->Priority;
 696          }
 697
 698          if (OverItem.get() == 0)
 699             OverItem = Itm;
 700       }
 701    }
 702
 703    // If we need to do any rewriting of the header do it now..
 704    if (OverItem.get() == 0)
 705    {
 706       if (NoOverride == false)
 707       {
 708          NewLine(1);
 709          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 710       }
 711
 712       OverItem = auto_ptr<Override::Item>(new Override::Item);
 713    }
 714
 715    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 716    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 717    if (SOverItem.get() == 0)
 718    {
 719       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 720       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 721       if (SOverItem.get() == 0)
 722       {
 723         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 724          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 725          *SOverItem = *OverItem;
 726       }
 727    }
 728
 729    // Add the dsc to the files hash list
 730    string const strippedName = flNotDir(FileName);
 731    std::ostringstream ostreamFiles;
 732    if (DoMD5 == true && Tags.Exists("Files"))
 733       ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 734                    << strippedName << "\n " << Tags.FindS("Files");
 735    string const Files = ostreamFiles.str();
 736
 737    std::ostringstream ostreamSha1;
 738    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 739       ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 740                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 741    string const ChecksumsSha1 = ostreamSha1.str();
 742
 743    std::ostringstream ostreamSha256;
 744    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 745       ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 746                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 747    string const ChecksumsSha256 = ostreamSha256.str();
 748
 749    std::ostringstream ostreamSha512;
 750    if (Tags.Exists("Checksums-Sha512"))
 751       ostreamSha512 << "\n " << string(SHA512.Result()) << " " << St.st_size << " "
 752                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 753    string const ChecksumsSha512 = ostreamSha512.str();
 754
 755    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 756    string NewFileName;
 757    if (DirStrip.empty() == false &&
 758        FileName.length() > DirStrip.length() &&
 759        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 760       NewFileName = string(OriginalPath + DirStrip.length());
 761    else
 762       NewFileName = OriginalPath;
 763    if (PathPrefix.empty() == false)
 764       NewFileName = flCombine(PathPrefix,NewFileName);
 765
 766    string Directory = flNotFile(OriginalPath);
 767    string Package = Tags.FindS("Source");
 768
 769    // Perform the delinking operation over all of the files
 770    string ParseJnk;
 771    const char *C = Files.c_str();
 772    char *RealPath = NULL;
 773    for (;isspace(*C); C++);
 774    while (*C != 0)
 775    {
 776       // Parse each of the elements
 777       if (ParseQuoteWord(C,ParseJnk) == false ||
 778           ParseQuoteWord(C,ParseJnk) == false ||
 779           ParseQuoteWord(C,ParseJnk) == false)
 780          return _error->Error("Error parsing file record");
 781
 782       char Jnk[2];
 783       string OriginalPath = Directory + ParseJnk;
 784       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 785           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 786       {
 787          string RP = RealPath;
 788          free(RealPath);
 789          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 790             return false;
 791       }
 792    }
 793
 794    Directory = flNotFile(NewFileName);
 795    if (Directory.length() > 2)
 796       Directory.erase(Directory.end()-1);
 797
 798    // This lists all the changes to the fields we are going to make.
 799    // (5 hardcoded + checksums + maintainer + end marker)
 800    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 801
 802    unsigned int End = 0;
 803    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 804    if (Files.empty() == false)
 805       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 806    if (ChecksumsSha1.empty() == false)
 807       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 808    if (ChecksumsSha256.empty() == false)
 809       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 810    if (ChecksumsSha512.empty() == false)
 811       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 812    if (Directory != "./")
 813       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 814    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 815    SetTFRewriteData(Changes[End++],"Status",0);
 816
 817    // Rewrite the maintainer field if necessary
 818    bool MaintFailed;
 819    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 820    if (MaintFailed == true)
 821    {
 822       if (NoOverride == false)
 823       {
 824          NewLine(1);
 825          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 826                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 827       }
 828    }
 829    if (NewMaint.empty() == false)
 830       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 831
 832    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 833         I != SOverItem->FieldOverride.end(); ++I)
 834       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 835
 836    SetTFRewriteData(Changes[End++], 0, 0);
 837
 838    // Rewrite and store the fields.
 839    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 840       return false;
 841    fprintf(Output,"\n");
 842
 843    Stats.Packages++;
 844
 845    return true;
 846 }
 847                                                                         /*}}}*/
 848
 849 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 850 // ---------------------------------------------------------------------
 851 /* */
 852 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 853                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 854
 855 {
 856    SetExts(".deb");
 857    Output = stdout;
 858 }
 859                                                                         /*}}}*/
 860 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 861 // ---------------------------------------------------------------------
 862 /* If Package is the empty string the control record will be parsed to
 863    determine what the package name is. */
 864 bool ContentsWriter::DoPackage(string FileName, string Package)
 865 {
 866    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 867    {
 868       return false;
 869    }
 870
 871    // Parse the package name
 872    if (Package.empty() == true)
 873    {
 874       Package = Db.Control.Section.FindS("Package");
 875    }
 876
 877    Db.Contents.Add(Gen,Package);
 878
 879    return Db.Finish();
 880 }
 881                                                                         /*}}}*/
 882 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 883 // ---------------------------------------------------------------------
 884 /* */
 885 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 886 {
 887    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 888    if (_error->PendingError() == true)
 889       return false;
 890
 891    // Open the package file
 892    int CompFd = -1;
 893    pid_t Proc = -1;
 894    if (Pkgs.OpenOld(CompFd,Proc) == false)
 895       return false;
 896
 897    // No auto-close FD
 898    FileFd Fd(CompFd,false);
 899    pkgTagFile Tags(&Fd);
 900    if (_error->PendingError() == true)
 901    {
 902       Pkgs.CloseOld(CompFd,Proc);
 903       return false;
 904    }
 905
 906    // Parse.
 907    pkgTagSection Section;
 908    while (Tags.Step(Section) == true)
 909    {
 910       string File = flCombine(Prefix,Section.FindS("FileName"));
 911       string Package = Section.FindS("Section");
 912       if (Package.empty() == false && Package.end()[-1] != '/')
 913       {
 914          Package += '/';
 915          Package += Section.FindS("Package");
 916       }
 917       else
 918          Package += Section.FindS("Package");
 919
 920       DoPackage(File,Package);
 921       if (_error->empty() == false)
 922       {
 923          _error->Error("Errors apply to file '%s'",File.c_str());
 924          _error->DumpErrors();
 925       }
 926    }
 927
 928    // Tidy the compressor
 929    if (Pkgs.CloseOld(CompFd,Proc) == false)
 930       return false;
 931
 932    return true;
 933 }
 934
 935                                                                         /*}}}*/
 936
 937 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 938 // ---------------------------------------------------------------------
 939 /* */
 940 ReleaseWriter::ReleaseWriter(string const &DB)
 941 {
 942    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 943    {
 944       AddPattern("Packages");
 945       AddPattern("Packages.gz");
 946       AddPattern("Packages.bz2");
 947       AddPattern("Packages.lzma");
 948       AddPattern("Packages.xz");
 949       AddPattern("Sources");
 950       AddPattern("Sources.gz");
 951       AddPattern("Sources.bz2");
 952       AddPattern("Sources.lzma");
 953       AddPattern("Sources.xz");
 954       AddPattern("Release");
 955       AddPattern("Index");
 956       AddPattern("md5sum.txt");
 957    }
 958    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 959
 960    Output = stdout;
 961    time_t const now = time(NULL);
 962
 963    setlocale(LC_TIME, "C");
 964
 965    char datestr[128];
 966    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 967                 gmtime(&now)) == 0)
 968    {
 969       datestr[0] = '\0';
 970    }
 971
 972    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 973    char validstr[128];
 974    if (now == validuntil ||
 975        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 976                 gmtime(&validuntil)) == 0)
 977    {
 978       validstr[0] = '\0';
 979    }
 980
 981    setlocale(LC_TIME, "");
 982
 983    map<string,string> Fields;
 984    Fields["Origin"] = "";
 985    Fields["Label"] = "";
 986    Fields["Suite"] = "";
 987    Fields["Version"] = "";
 988    Fields["Codename"] = "";
 989    Fields["Date"] = datestr;
 990    Fields["Valid-Until"] = validstr;
 991    Fields["Architectures"] = "";
 992    Fields["Components"] = "";
 993    Fields["Description"] = "";
 994
 995    for(map<string,string>::const_iterator I = Fields.begin();
 996        I != Fields.end();
 997        ++I)
 998    {
 999       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1000       string Value = _config->Find(Config, (*I).second.c_str());
1001       if (Value == "")
1002          continue;
1003
1004       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1005    }
1006
1007    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1008    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1009    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1010 }
1011                                                                         /*}}}*/
1012 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1013 // ---------------------------------------------------------------------
1014 bool ReleaseWriter::DoPackage(string FileName)
1015 {
1016    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1017    string NewFileName;
1018    if (DirStrip.empty() == false &&
1019        FileName.length() > DirStrip.length() &&
1020        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1021                  DirStrip.begin(),DirStrip.end()) == 0)
1022    {
1023       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1024       while (NewFileName[0] == '/')
1025          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1026    }
1027    else
1028       NewFileName = FileName;
1029
1030    if (PathPrefix.empty() == false)
1031       NewFileName = flCombine(PathPrefix,NewFileName);
1032
1033    FileFd fd(FileName, FileFd::ReadOnly);
1034
1035    if (!fd.IsOpen())
1036    {
1037       return false;
1038    }
1039
1040    CheckSums[NewFileName].size = fd.Size();
1041
1042    Hashes hs;
1043    hs.AddFD(fd.Fd(), 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1044    if (DoMD5 == true)
1045       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1046    if (DoSHA1 == true)
1047       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1048    if (DoSHA256 == true)
1049       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1050    if (DoSHA512 == true)
1051       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1052    fd.Close();
1053
1054    return true;
1055 }
1056
1057                                                                         /*}}}*/
1058 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1059 // ---------------------------------------------------------------------
1060 void ReleaseWriter::Finish()
1061 {
1062    if (DoMD5 == true)
1063    {
1064       fprintf(Output, "MD5Sum:\n");
1065       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1066           I != CheckSums.end(); ++I)
1067       {
1068          fprintf(Output, " %s %16ld %s\n",
1069                  (*I).second.MD5.c_str(),
1070                  (*I).second.size,
1071                  (*I).first.c_str());
1072       }
1073    }
1074    if (DoSHA1 == true)
1075    {
1076       fprintf(Output, "SHA1:\n");
1077       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1078           I != CheckSums.end(); ++I)
1079       {
1080          fprintf(Output, " %s %16ld %s\n",
1081                  (*I).second.SHA1.c_str(),
1082                  (*I).second.size,
1083                  (*I).first.c_str());
1084       }
1085    }
1086    if (DoSHA256 == true)
1087    {
1088       fprintf(Output, "SHA256:\n");
1089       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1090           I != CheckSums.end(); ++I)
1091       {
1092          fprintf(Output, " %s %16ld %s\n",
1093                  (*I).second.SHA256.c_str(),
1094                  (*I).second.size,
1095                  (*I).first.c_str());
1096       }
1097    }
1098
1099    fprintf(Output, "SHA512:\n");
1100    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1101        I != CheckSums.end();
1102        ++I)
1103    {
1104       fprintf(Output, " %s %32ld %s\n",
1105               (*I).second.SHA512.c_str(),
1106               (*I).second.size,
1107               (*I).first.c_str());
1108    }
1109
1110 }