ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include "writer.h"
  15
  16 #include <apti18n.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/md5.h>
  22 #include <apt-pkg/sha1.h>
  23 #include <apt-pkg/sha2.h>
  24 #include <apt-pkg/deblistparser.h>
  25
  26 #include <sys/types.h>
  27 #include <unistd.h>
  28 #include <ctime>
  29 #include <ftw.h>
  30 #include <fnmatch.h>
  31 #include <iostream>
  32 #include <sstream>
  33 #include <memory>
  34
  35 #include "cachedb.h"
  36 #include "apt-ftparchive.h"
  37 #include "multicompress.h"
  38                                                                         /*}}}*/
  39 using namespace std;
  40 FTWScanner *FTWScanner::Owner;
  41
  42 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* */
  45 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  46                              const char *tag,
  47                              const char *rewrite,
  48                              const char *newtag = 0)
  49 {
  50     tfrd.Tag = tag;
  51     tfrd.Rewrite = rewrite;
  52     tfrd.NewTag = newtag;
  53 }
  54                                                                         /*}}}*/
  55
  56 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  57 // ---------------------------------------------------------------------
  58 /* */
  59 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  60 {
  61    ErrorPrinted = false;
  62    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  63
  64    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  65    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  66    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  67    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  68 }
  69                                                                         /*}}}*/
  70 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  71 // ---------------------------------------------------------------------
  72 /* This is the FTW scanner, it processes each directory element in the
  73    directory tree. */
  74 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  75 {
  76    if (Flag == FTW_DNR)
  77    {
  78       Owner->NewLine(1);
  79       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  80    }
  81    if (Flag == FTW_NS)
  82    {
  83       Owner->NewLine(1);
  84       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  85    }
  86    if (Flag != FTW_F)
  87       return 0;
  88
  89    return ScannerFile(File, true);
  90 }
  91                                                                         /*}}}*/
  92 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  93 // ---------------------------------------------------------------------
  94 /* */
  95 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  96 {
  97    const char *LastComponent = strrchr(File, '/');
  98    char *RealPath = NULL;
  99
 100    if (LastComponent == NULL)
 101       LastComponent = File;
 102    else
 103       LastComponent++;
 104
 105    vector<string>::const_iterator I;
 106    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 107    {
 108       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 109          break;
 110    }
 111    if (I == Owner->Patterns.end())
 112       return 0;
 113
 114    /* Process it. If the file is a link then resolve it into an absolute
 115       name.. This works best if the directory components the scanner are
 116       given are not links themselves. */
 117    char Jnk[2];
 118    Owner->OriginalPath = File;
 119    if (ReadLink &&
 120        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 121        (RealPath = realpath(File,NULL)) != 0)
 122    {
 123       Owner->DoPackage(RealPath);
 124       free(RealPath);
 125    }
 126    else
 127       Owner->DoPackage(File);
 128
 129    if (_error->empty() == false)
 130    {
 131       // Print any errors or warnings found
 132       string Err;
 133       bool SeenPath = false;
 134       while (_error->empty() == false)
 135       {
 136          Owner->NewLine(1);
 137
 138          bool const Type = _error->PopMessage(Err);
 139          if (Type == true)
 140             cerr << _("E: ") << Err << endl;
 141          else
 142             cerr << _("W: ") << Err << endl;
 143
 144          if (Err.find(File) != string::npos)
 145             SeenPath = true;
 146       }
 147
 148       if (SeenPath == false)
 149          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 150       return 0;
 151    }
 152
 153    return 0;
 154 }
 155                                                                         /*}}}*/
 156 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* */
 159 bool FTWScanner::RecursiveScan(string const &Dir)
 160 {
 161    char *RealPath = NULL;
 162    /* If noprefix is set then jam the scan root in, so we don't generate
 163       link followed paths out of control */
 164    if (InternalPrefix.empty() == true)
 165    {
 166       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 167          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 168       InternalPrefix = RealPath;
 169       free(RealPath);
 170    }
 171
 172    // Do recursive directory searching
 173    Owner = this;
 174    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 175
 176    // Error treewalking?
 177    if (Res != 0)
 178    {
 179       if (_error->PendingError() == false)
 180          _error->Errno("ftw",_("Tree walking failed"));
 181       return false;
 182    }
 183
 184    return true;
 185 }
 186                                                                         /*}}}*/
 187 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* This is an alternative to using FTW to locate files, it reads the list
 190    of files from another file. */
 191 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 192 {
 193    char *RealPath = NULL;
 194    /* If noprefix is set then jam the scan root in, so we don't generate
 195       link followed paths out of control */
 196    if (InternalPrefix.empty() == true)
 197    {
 198       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 199          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 200       InternalPrefix = RealPath;
 201       free(RealPath);
 202    }
 203
 204    Owner = this;
 205    FILE *List = fopen(File.c_str(),"r");
 206    if (List == 0)
 207       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 208
 209    /* We are a tad tricky here.. We prefix the buffer with the directory
 210       name, that way if we need a full path with just use line.. Sneaky and
 211       fully evil. */
 212    char Line[1000];
 213    char *FileStart;
 214    if (Dir.empty() == true || Dir.end()[-1] != '/')
 215       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 216    else
 217       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 218    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 219    {
 220       char *FileName = _strstrip(FileStart);
 221       if (FileName[0] == 0)
 222          continue;
 223
 224       if (FileName[0] != '/')
 225       {
 226          if (FileName != FileStart)
 227             memmove(FileStart,FileName,strlen(FileStart));
 228          FileName = Line;
 229       }
 230
 231 #if 0
 232       struct stat St;
 233       int Flag = FTW_F;
 234       if (stat(FileName,&St) != 0)
 235          Flag = FTW_NS;
 236 #endif
 237
 238       if (ScannerFile(FileName, false) != 0)
 239          break;
 240    }
 241
 242    fclose(List);
 243    return true;
 244 }
 245                                                                         /*}}}*/
 246 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 247 // ---------------------------------------------------------------------
 248 /* */
 249 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 250                         unsigned long &DeLinkBytes,
 251                         off_t const &FileSize)
 252 {
 253    // See if this isn't an internaly prefix'd file name.
 254    if (InternalPrefix.empty() == false &&
 255        InternalPrefix.length() < FileName.length() &&
 256        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 257                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 258    {
 259       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 260       {
 261          // Tidy up the display
 262          if (DeLinkBytes == 0)
 263             cout << endl;
 264
 265          NewLine(1);
 266          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 267                     SizeToStr(FileSize).c_str());
 268          c1out << flush;
 269
 270          if (NoLinkAct == false)
 271          {
 272             char OldLink[400];
 273             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 274                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 275             else
 276             {
 277                if (unlink(OriginalPath) != 0)
 278                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 279                else
 280                {
 281                   if (link(FileName.c_str(),OriginalPath) != 0)
 282                   {
 283                      // Panic! Restore the symlink
 284                      symlink(OldLink,OriginalPath);
 285                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 286                                           FileName.c_str(),
 287                                           OriginalPath);
 288                   }
 289                }
 290             }
 291          }
 292
 293          DeLinkBytes += FileSize;
 294          if (DeLinkBytes/1024 >= DeLinkLimit)
 295             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 296       }
 297
 298       FileName = OriginalPath;
 299    }
 300
 301    return true;
 302 }
 303                                                                         /*}}}*/
 304
 305 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 306 // ---------------------------------------------------------------------
 307 /* */
 308 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 309                                string const &Arch) :
 310    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 311 {
 312    Output = stdout;
 313    SetExts(".deb .udeb");
 314    DeLinkLimit = 0;
 315
 316    // Process the command line options
 317    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 318    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 319    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 320    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA512",true);
 321    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 322    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 323    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 324    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 325
 326    if (Db.Loaded() == false)
 327       DoContents = false;
 328
 329    // Read the override file
 330    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 331       return;
 332    else
 333       NoOverride = true;
 334
 335    if (ExtOverrides.empty() == false)
 336       Over.ReadExtraOverride(ExtOverrides);
 337
 338    _error->DumpErrors();
 339 }
 340                                                                         /*}}}*/
 341 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* */
 344 bool FTWScanner::SetExts(string const &Vals)
 345 {
 346    ClearPatterns();
 347    string::size_type Start = 0;
 348    while (Start <= Vals.length()-1)
 349    {
 350       string::size_type const Space = Vals.find(' ',Start);
 351       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 352       if ( Arch.empty() == false )
 353       {
 354          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 355          AddPattern(string("*_all") + Vals.substr(Start, Length));
 356       }
 357       else
 358          AddPattern(string("*") + Vals.substr(Start, Length));
 359
 360       Start += Length + 1;
 361    }
 362
 363    return true;
 364 }
 365
 366                                                                         /*}}}*/
 367 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* This method takes a package and gets its control information and
 370    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 371    rewritten and the path/size/hash appended. */
 372 bool PackagesWriter::DoPackage(string FileName)
 373 {
 374    // Pull all the data we need form the DB
 375    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 376                   == false)
 377    {
 378       return false;
 379    }
 380
 381    off_t FileSize = Db.GetFileSize();
 382    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 383       return false;
 384
 385    // Lookup the overide information
 386    pkgTagSection &Tags = Db.Control.Section;
 387    string Package = Tags.FindS("Package");
 388    string Architecture;
 389    // if we generate a Packages file for a given arch, we use it to
 390    // look for overrides. if we run in "simple" mode without the
 391    // "Architecures" variable in the config we use the architecure value
 392    // from the deb file
 393    if(Arch != "")
 394       Architecture = Arch;
 395    else
 396       Architecture = Tags.FindS("Architecture");
 397    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 398
 399    if (Package.empty() == true)
 400       return _error->Error(_("Archive had no package field"));
 401
 402    // If we need to do any rewriting of the header do it now..
 403    if (OverItem.get() == 0)
 404    {
 405       if (NoOverride == false)
 406       {
 407          NewLine(1);
 408          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 409       }
 410
 411       OverItem = auto_ptr<Override::Item>(new Override::Item);
 412       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 413       OverItem->Priority = Tags.FindS("Priority");
 414    }
 415
 416    char Size[40];
 417    sprintf(Size,"%lu", (unsigned long) FileSize);
 418
 419    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 420    string NewFileName;
 421    if (DirStrip.empty() == false &&
 422        FileName.length() > DirStrip.length() &&
 423        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 424                  DirStrip.begin(),DirStrip.end()) == 0)
 425       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 426    else
 427       NewFileName = FileName;
 428    if (PathPrefix.empty() == false)
 429       NewFileName = flCombine(PathPrefix,NewFileName);
 430
 431    /* Configuration says we don't want to include the long Description
 432       in the package file - instead we want to ship a separated file */
 433    string desc;
 434    if (LongDescription == false) {
 435       desc = Tags.FindS("Description").append("\n");
 436       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 437    }
 438
 439    // This lists all the changes to the fields we are going to make.
 440    // (7 hardcoded + maintainer + suggests + end marker)
 441    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 442
 443    unsigned int End = 0;
 444    SetTFRewriteData(Changes[End++], "Size", Size);
 445    if (DoMD5 == true)
 446       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 447    if (DoSHA1 == true)
 448       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 449    if (DoSHA256 == true)
 450       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 451    if (DoSHA512 == true)
 452       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 453    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 454    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 455    SetTFRewriteData(Changes[End++], "Status", 0);
 456    SetTFRewriteData(Changes[End++], "Optional", 0);
 457
 458    string DescriptionMd5;
 459    if (LongDescription == false) {
 460       MD5Summation descmd5;
 461       descmd5.Add(desc.c_str());
 462       DescriptionMd5 = descmd5.Result().Value();
 463       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 464       if (TransWriter != NULL)
 465          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 466    }
 467
 468    // Rewrite the maintainer field if necessary
 469    bool MaintFailed;
 470    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 471    if (MaintFailed == true)
 472    {
 473       if (NoOverride == false)
 474       {
 475          NewLine(1);
 476          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 477                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 478       }
 479    }
 480
 481    if (NewMaint.empty() == false)
 482       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 483
 484    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 485       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 486       but dpkg does this append bit. So we do the append bit, at least that way the
 487       status file and package file will remain similar. There are other transforms
 488       but optional is the only legacy one still in use for some lazy reason. */
 489    string OptionalStr = Tags.FindS("Optional");
 490    if (OptionalStr.empty() == false)
 491    {
 492       if (Tags.FindS("Suggests").empty() == false)
 493          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 494       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 495    }
 496
 497    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 498         I != OverItem->FieldOverride.end(); I++)
 499       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 500
 501    SetTFRewriteData(Changes[End++], 0, 0);
 502
 503    // Rewrite and store the fields.
 504    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 505       return false;
 506    fprintf(Output,"\n");
 507
 508    return Db.Finish();
 509 }
 510                                                                         /*}}}*/
 511
 512 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 513 // ---------------------------------------------------------------------
 514 /* Create a Translation-Master file for this Packages file */
 515 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 516                                         mode_t const &Permissions) : Output(NULL),
 517                                                         RefCounter(0)
 518 {
 519    if (File.empty() == true)
 520       return;
 521
 522    Comp = new MultiCompress(File, TransCompress, Permissions);
 523    Output = Comp->Input;
 524 }
 525                                                                         /*}}}*/
 526 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 527 // ---------------------------------------------------------------------
 528 /* Create a Translation-Master file for this Packages file */
 529 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 530                                   string const &MD5)
 531 {
 532    if (Output == NULL)
 533       return true;
 534
 535    // Different archs can include different versions and therefore
 536    // different descriptions - so we need to check for both name and md5.
 537    string const Record = Pkg + ":" + MD5;
 538
 539    if (Included.find(Record) != Included.end())
 540       return true;
 541
 542    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 543            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 544
 545    Included.insert(Record);
 546    return true;
 547 }
 548                                                                         /*}}}*/
 549 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 550 // ---------------------------------------------------------------------
 551 /* */
 552 TranslationWriter::~TranslationWriter()
 553 {
 554    if (Comp == NULL)
 555       return;
 556
 557    delete Comp;
 558 }
 559                                                                         /*}}}*/
 560
 561 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 562 // ---------------------------------------------------------------------
 563 /* */
 564 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 565                              string const &ExtOverrides)
 566 {
 567    Output = stdout;
 568    AddPattern("*.dsc");
 569    DeLinkLimit = 0;
 570    Buffer = 0;
 571    BufSize = 0;
 572
 573    // Process the command line options
 574    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 575    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 576    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 577    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 578
 579    // Read the override file
 580    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 581       return;
 582    else
 583       NoOverride = true;
 584
 585    // WTF?? The logic above: if we can't read binary overrides, don't even try
 586    // reading source overrides. if we can read binary overrides, then say there
 587    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 588
 589    if (ExtOverrides.empty() == false)
 590       SOver.ReadExtraOverride(ExtOverrides);
 591
 592    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 593       SOver.ReadOverride(SOverrides,true);
 594 }
 595                                                                         /*}}}*/
 596 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 597 // ---------------------------------------------------------------------
 598 /* */
 599 bool SourcesWriter::DoPackage(string FileName)
 600 {
 601    // Open the archive
 602    FileFd F(FileName,FileFd::ReadOnly);
 603    if (_error->PendingError() == true)
 604       return false;
 605
 606    // Stat the file for later
 607    struct stat St;
 608    if (fstat(F.Fd(),&St) != 0)
 609       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 610
 611    if (St.st_size > 128*1024)
 612       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 613
 614    if (BufSize < (unsigned)St.st_size+1)
 615    {
 616       BufSize = St.st_size+1;
 617       Buffer = (char *)realloc(Buffer,St.st_size+1);
 618    }
 619
 620    if (F.Read(Buffer,St.st_size) == false)
 621       return false;
 622
 623    // Hash the file
 624    char *Start = Buffer;
 625    char *BlkEnd = Buffer + St.st_size;
 626
 627    MD5Summation MD5;
 628    SHA1Summation SHA1;
 629    SHA256Summation SHA256;
 630    SHA256Summation SHA512;
 631
 632    if (DoMD5 == true)
 633       MD5.Add((unsigned char *)Start,BlkEnd - Start);
 634    if (DoSHA1 == true)
 635       SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 636    if (DoSHA256 == true)
 637       SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 638    if (DoSHA512 == true)
 639       SHA512.Add((unsigned char *)Start,BlkEnd - Start);
 640
 641    // Add an extra \n to the end, just in case
 642    *BlkEnd++ = '\n';
 643
 644    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 645       before */
 646    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 647    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 648    {
 649       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 650       {
 651          MsgEnd[1] = '\n';
 652          break;
 653       }
 654    }
 655
 656    /* Read records until we locate the Source record. This neatly skips the
 657       GPG header (which is RFC822 formed) without any trouble. */
 658    pkgTagSection Tags;
 659    do
 660    {
 661       unsigned Pos;
 662       if (Tags.Scan(Start,BlkEnd - Start) == false)
 663          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 664       if (Tags.Find("Source",Pos) == true)
 665          break;
 666       Start += Tags.size();
 667    }
 668    while (1);
 669    Tags.Trim();
 670
 671    // Lookup the overide information, finding first the best priority.
 672    string BestPrio;
 673    string Bins = Tags.FindS("Binary");
 674    char Buffer[Bins.length() + 1];
 675    auto_ptr<Override::Item> OverItem(0);
 676    if (Bins.empty() == false)
 677    {
 678       strcpy(Buffer,Bins.c_str());
 679
 680       // Ignore too-long errors.
 681       char *BinList[400];
 682       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 683
 684       // Look at all the binaries
 685       unsigned char BestPrioV = pkgCache::State::Extra;
 686       for (unsigned I = 0; BinList[I] != 0; I++)
 687       {
 688          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 689          if (Itm.get() == 0)
 690             continue;
 691
 692          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 693          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 694          {
 695             BestPrioV = NewPrioV;
 696             BestPrio = Itm->Priority;
 697          }
 698
 699          if (OverItem.get() == 0)
 700             OverItem = Itm;
 701       }
 702    }
 703
 704    // If we need to do any rewriting of the header do it now..
 705    if (OverItem.get() == 0)
 706    {
 707       if (NoOverride == false)
 708       {
 709          NewLine(1);
 710          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 711       }
 712
 713       OverItem = auto_ptr<Override::Item>(new Override::Item);
 714    }
 715
 716    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 717    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 718    if (SOverItem.get() == 0)
 719    {
 720       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 721       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 722       if (SOverItem.get() == 0)
 723       {
 724         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 725          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 726          *SOverItem = *OverItem;
 727       }
 728    }
 729
 730    // Add the dsc to the files hash list
 731    string const strippedName = flNotDir(FileName);
 732    std::ostringstream ostreamFiles;
 733    if (DoMD5 == true && Tags.Exists("Files"))
 734       ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 735                    << strippedName << "\n " << Tags.FindS("Files");
 736    string const Files = ostreamFiles.str();
 737
 738    std::ostringstream ostreamSha1;
 739    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 740       ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 741                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 742    string const ChecksumsSha1 = ostreamSha1.str();
 743
 744    std::ostringstream ostreamSha256;
 745    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 746       ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 747                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 748    string const ChecksumsSha256 = ostreamSha256.str();
 749
 750    std::ostringstream ostreamSha512;
 751    if (Tags.Exists("Checksums-Sha512"))
 752       ostreamSha512 << "\n " << string(SHA512.Result()) << " " << St.st_size << " "
 753                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 754    string const ChecksumsSha512 = ostreamSha512.str();
 755
 756    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 757    string NewFileName;
 758    if (DirStrip.empty() == false &&
 759        FileName.length() > DirStrip.length() &&
 760        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 761       NewFileName = string(OriginalPath + DirStrip.length());
 762    else
 763       NewFileName = OriginalPath;
 764    if (PathPrefix.empty() == false)
 765       NewFileName = flCombine(PathPrefix,NewFileName);
 766
 767    string Directory = flNotFile(OriginalPath);
 768    string Package = Tags.FindS("Source");
 769
 770    // Perform the delinking operation over all of the files
 771    string ParseJnk;
 772    const char *C = Files.c_str();
 773    char *RealPath = NULL;
 774    for (;isspace(*C); C++);
 775    while (*C != 0)
 776    {
 777       // Parse each of the elements
 778       if (ParseQuoteWord(C,ParseJnk) == false ||
 779           ParseQuoteWord(C,ParseJnk) == false ||
 780           ParseQuoteWord(C,ParseJnk) == false)
 781          return _error->Error("Error parsing file record");
 782
 783       char Jnk[2];
 784       string OriginalPath = Directory + ParseJnk;
 785       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 786           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 787       {
 788          string RP = RealPath;
 789          free(RealPath);
 790          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 791             return false;
 792       }
 793    }
 794
 795    Directory = flNotFile(NewFileName);
 796    if (Directory.length() > 2)
 797       Directory.erase(Directory.end()-1);
 798
 799    // This lists all the changes to the fields we are going to make.
 800    // (5 hardcoded + checksums + maintainer + end marker)
 801    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 802
 803    unsigned int End = 0;
 804    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 805    if (Files.empty() == false)
 806       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 807    if (ChecksumsSha1.empty() == false)
 808       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 809    if (ChecksumsSha256.empty() == false)
 810       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 811    if (ChecksumsSha512.empty() == false)
 812       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 813    if (Directory != "./")
 814       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 815    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 816    SetTFRewriteData(Changes[End++],"Status",0);
 817
 818    // Rewrite the maintainer field if necessary
 819    bool MaintFailed;
 820    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 821    if (MaintFailed == true)
 822    {
 823       if (NoOverride == false)
 824       {
 825          NewLine(1);
 826          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 827                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 828       }
 829    }
 830    if (NewMaint.empty() == false)
 831       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 832
 833    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 834         I != SOverItem->FieldOverride.end(); I++)
 835       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 836
 837    SetTFRewriteData(Changes[End++], 0, 0);
 838
 839    // Rewrite and store the fields.
 840    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 841       return false;
 842    fprintf(Output,"\n");
 843
 844    Stats.Packages++;
 845
 846    return true;
 847 }
 848                                                                         /*}}}*/
 849
 850 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 851 // ---------------------------------------------------------------------
 852 /* */
 853 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 854                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 855
 856 {
 857    SetExts(".deb");
 858    Output = stdout;
 859 }
 860                                                                         /*}}}*/
 861 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 862 // ---------------------------------------------------------------------
 863 /* If Package is the empty string the control record will be parsed to
 864    determine what the package name is. */
 865 bool ContentsWriter::DoPackage(string FileName, string Package)
 866 {
 867    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 868    {
 869       return false;
 870    }
 871
 872    // Parse the package name
 873    if (Package.empty() == true)
 874    {
 875       Package = Db.Control.Section.FindS("Package");
 876    }
 877
 878    Db.Contents.Add(Gen,Package);
 879
 880    return Db.Finish();
 881 }
 882                                                                         /*}}}*/
 883 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 884 // ---------------------------------------------------------------------
 885 /* */
 886 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 887 {
 888    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 889    if (_error->PendingError() == true)
 890       return false;
 891
 892    // Open the package file
 893    int CompFd = -1;
 894    pid_t Proc = -1;
 895    if (Pkgs.OpenOld(CompFd,Proc) == false)
 896       return false;
 897
 898    // No auto-close FD
 899    FileFd Fd(CompFd,false);
 900    pkgTagFile Tags(&Fd);
 901    if (_error->PendingError() == true)
 902    {
 903       Pkgs.CloseOld(CompFd,Proc);
 904       return false;
 905    }
 906
 907    // Parse.
 908    pkgTagSection Section;
 909    while (Tags.Step(Section) == true)
 910    {
 911       string File = flCombine(Prefix,Section.FindS("FileName"));
 912       string Package = Section.FindS("Section");
 913       if (Package.empty() == false && Package.end()[-1] != '/')
 914       {
 915          Package += '/';
 916          Package += Section.FindS("Package");
 917       }
 918       else
 919          Package += Section.FindS("Package");
 920
 921       DoPackage(File,Package);
 922       if (_error->empty() == false)
 923       {
 924          _error->Error("Errors apply to file '%s'",File.c_str());
 925          _error->DumpErrors();
 926       }
 927    }
 928
 929    // Tidy the compressor
 930    if (Pkgs.CloseOld(CompFd,Proc) == false)
 931       return false;
 932
 933    return true;
 934 }
 935
 936                                                                         /*}}}*/
 937
 938 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 939 // ---------------------------------------------------------------------
 940 /* */
 941 ReleaseWriter::ReleaseWriter(string const &DB)
 942 {
 943    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 944    {
 945       AddPattern("Packages");
 946       AddPattern("Packages.gz");
 947       AddPattern("Packages.bz2");
 948       AddPattern("Packages.lzma");
 949       AddPattern("Packages.xz");
 950       AddPattern("Sources");
 951       AddPattern("Sources.gz");
 952       AddPattern("Sources.bz2");
 953       AddPattern("Sources.lzma");
 954       AddPattern("Sources.xz");
 955       AddPattern("Release");
 956       AddPattern("Index");
 957       AddPattern("md5sum.txt");
 958    }
 959    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 960
 961    Output = stdout;
 962    time_t const now = time(NULL);
 963
 964    setlocale(LC_TIME, "C");
 965
 966    char datestr[128];
 967    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 968                 gmtime(&now)) == 0)
 969    {
 970       datestr[0] = '\0';
 971    }
 972
 973    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 974    char validstr[128];
 975    if (now == validuntil ||
 976        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 977                 gmtime(&validuntil)) == 0)
 978    {
 979       validstr[0] = '\0';
 980    }
 981
 982    setlocale(LC_TIME, "");
 983
 984    map<string,string> Fields;
 985    Fields["Origin"] = "";
 986    Fields["Label"] = "";
 987    Fields["Suite"] = "";
 988    Fields["Version"] = "";
 989    Fields["Codename"] = "";
 990    Fields["Date"] = datestr;
 991    Fields["Valid-Until"] = validstr;
 992    Fields["Architectures"] = "";
 993    Fields["Components"] = "";
 994    Fields["Description"] = "";
 995
 996    for(map<string,string>::const_iterator I = Fields.begin();
 997        I != Fields.end();
 998        ++I)
 999    {
1000       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1001       string Value = _config->Find(Config, (*I).second.c_str());
1002       if (Value == "")
1003          continue;
1004
1005       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1006    }
1007
1008    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1009    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1010    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1011 }
1012                                                                         /*}}}*/
1013 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1014 // ---------------------------------------------------------------------
1015 bool ReleaseWriter::DoPackage(string FileName)
1016 {
1017    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1018    string NewFileName;
1019    if (DirStrip.empty() == false &&
1020        FileName.length() > DirStrip.length() &&
1021        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1022                  DirStrip.begin(),DirStrip.end()) == 0)
1023    {
1024       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1025       while (NewFileName[0] == '/')
1026          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1027    }
1028    else
1029       NewFileName = FileName;
1030
1031    if (PathPrefix.empty() == false)
1032       NewFileName = flCombine(PathPrefix,NewFileName);
1033
1034    FileFd fd(FileName, FileFd::ReadOnly);
1035
1036    if (!fd.IsOpen())
1037    {
1038       return false;
1039    }
1040
1041    CheckSums[NewFileName].size = fd.Size();
1042
1043    if (DoMD5 == true)
1044    {
1045       MD5Summation MD5;
1046       MD5.AddFD(fd.Fd(), fd.Size());
1047       CheckSums[NewFileName].MD5 = MD5.Result();
1048       fd.Seek(0);
1049    }
1050    if (DoSHA1 == true)
1051    {
1052       SHA1Summation SHA1;
1053       SHA1.AddFD(fd.Fd(), fd.Size());
1054       CheckSums[NewFileName].SHA1 = SHA1.Result();
1055       fd.Seek(0);
1056    }
1057    if (DoSHA256 == true)
1058    {
1059       SHA256Summation SHA256;
1060       SHA256.AddFD(fd.Fd(), fd.Size());
1061       CheckSums[NewFileName].SHA256 = SHA256.Result();
1062    }
1063
1064    if (DoSHA512 == true)
1065    {
1066       SHA512Summation SHA512;
1067       SHA512.AddFD(fd.Fd(), fd.Size());
1068       CheckSums[NewFileName].SHA512 = SHA512.Result();
1069    }
1070    fd.Close();
1071
1072    return true;
1073 }
1074
1075                                                                         /*}}}*/
1076 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1077 // ---------------------------------------------------------------------
1078 void ReleaseWriter::Finish()
1079 {
1080    if (DoMD5 == true)
1081    {
1082       fprintf(Output, "MD5Sum:\n");
1083       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1084           I != CheckSums.end(); ++I)
1085       {
1086          fprintf(Output, " %s %16ld %s\n",
1087                  (*I).second.MD5.c_str(),
1088                  (*I).second.size,
1089                  (*I).first.c_str());
1090       }
1091    }
1092    if (DoSHA1 == true)
1093    {
1094       fprintf(Output, "SHA1:\n");
1095       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1096           I != CheckSums.end(); ++I)
1097       {
1098          fprintf(Output, " %s %16ld %s\n",
1099                  (*I).second.SHA1.c_str(),
1100                  (*I).second.size,
1101                  (*I).first.c_str());
1102       }
1103    }
1104    if (DoSHA256 == true)
1105    {
1106       fprintf(Output, "SHA256:\n");
1107       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1108           I != CheckSums.end(); ++I)
1109       {
1110          fprintf(Output, " %s %16ld %s\n",
1111                  (*I).second.SHA256.c_str(),
1112                  (*I).second.size,
1113                  (*I).first.c_str());
1114       }
1115    }
1116
1117    fprintf(Output, "SHA512:\n");
1118    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1119        I != CheckSums.end();
1120        ++I)
1121    {
1122       fprintf(Output, " %s %32ld %s\n",
1123               (*I).second.SHA512.c_str(),
1124               (*I).second.size,
1125               (*I).first.c_str());
1126    }
1127
1128 }