ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/error.h>
  18 #include <apt-pkg/configuration.h>
  19 #include <apt-pkg/aptconfiguration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/deblistparser.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/gpgv.h>
  25
  26 #include <sys/types.h>
  27 #include <unistd.h>
  28 #include <ctime>
  29 #include <ftw.h>
  30 #include <fnmatch.h>
  31 #include <iostream>
  32 #include <sstream>
  33 #include <memory>
  34
  35 #include "writer.h"
  36 #include "cachedb.h"
  37 #include "apt-ftparchive.h"
  38 #include "multicompress.h"
  39
  40 #include <apti18n.h>
  41                                                                         /*}}}*/
  42 using namespace std;
  43 FTWScanner *FTWScanner::Owner;
  44
  45 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  46 // ---------------------------------------------------------------------
  47 /* */
  48 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  49                              const char *tag,
  50                              const char *rewrite,
  51                              const char *newtag = 0)
  52 {
  53     tfrd.Tag = tag;
  54     tfrd.Rewrite = rewrite;
  55     tfrd.NewTag = newtag;
  56 }
  57                                                                         /*}}}*/
  58
  59 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  63 {
  64    ErrorPrinted = false;
  65    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  66
  67    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  68    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  69    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  70    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  71 }
  72                                                                         /*}}}*/
  73 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  74 // ---------------------------------------------------------------------
  75 /* This is the FTW scanner, it processes each directory element in the
  76    directory tree. */
  77 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  78 {
  79    if (Flag == FTW_DNR)
  80    {
  81       Owner->NewLine(1);
  82       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  83    }
  84    if (Flag == FTW_NS)
  85    {
  86       Owner->NewLine(1);
  87       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  88    }
  89    if (Flag != FTW_F)
  90       return 0;
  91
  92    return ScannerFile(File, true);
  93 }
  94                                                                         /*}}}*/
  95 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* */
  98 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  99 {
 100    const char *LastComponent = strrchr(File, '/');
 101    char *RealPath = NULL;
 102
 103    if (LastComponent == NULL)
 104       LastComponent = File;
 105    else
 106       LastComponent++;
 107
 108    vector<string>::const_iterator I;
 109    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 110    {
 111       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 112          break;
 113    }
 114    if (I == Owner->Patterns.end())
 115       return 0;
 116
 117    /* Process it. If the file is a link then resolve it into an absolute
 118       name.. This works best if the directory components the scanner are
 119       given are not links themselves. */
 120    char Jnk[2];
 121    Owner->OriginalPath = File;
 122    if (ReadLink &&
 123        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 124        (RealPath = realpath(File,NULL)) != 0)
 125    {
 126       Owner->DoPackage(RealPath);
 127       free(RealPath);
 128    }
 129    else
 130       Owner->DoPackage(File);
 131
 132    if (_error->empty() == false)
 133    {
 134       // Print any errors or warnings found
 135       string Err;
 136       bool SeenPath = false;
 137       while (_error->empty() == false)
 138       {
 139          Owner->NewLine(1);
 140
 141          bool const Type = _error->PopMessage(Err);
 142          if (Type == true)
 143             cerr << _("E: ") << Err << endl;
 144          else
 145             cerr << _("W: ") << Err << endl;
 146
 147          if (Err.find(File) != string::npos)
 148             SeenPath = true;
 149       }
 150
 151       if (SeenPath == false)
 152          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 153       return 0;
 154    }
 155
 156    return 0;
 157 }
 158                                                                         /*}}}*/
 159 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 160 // ---------------------------------------------------------------------
 161 /* */
 162 bool FTWScanner::RecursiveScan(string const &Dir)
 163 {
 164    char *RealPath = NULL;
 165    /* If noprefix is set then jam the scan root in, so we don't generate
 166       link followed paths out of control */
 167    if (InternalPrefix.empty() == true)
 168    {
 169       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 170          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 171       InternalPrefix = RealPath;
 172       free(RealPath);
 173    }
 174
 175    // Do recursive directory searching
 176    Owner = this;
 177    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 178
 179    // Error treewalking?
 180    if (Res != 0)
 181    {
 182       if (_error->PendingError() == false)
 183          _error->Errno("ftw",_("Tree walking failed"));
 184       return false;
 185    }
 186
 187    return true;
 188 }
 189                                                                         /*}}}*/
 190 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 191 // ---------------------------------------------------------------------
 192 /* This is an alternative to using FTW to locate files, it reads the list
 193    of files from another file. */
 194 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 195 {
 196    char *RealPath = NULL;
 197    /* If noprefix is set then jam the scan root in, so we don't generate
 198       link followed paths out of control */
 199    if (InternalPrefix.empty() == true)
 200    {
 201       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 202          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 203       InternalPrefix = RealPath;
 204       free(RealPath);
 205    }
 206
 207    Owner = this;
 208    FILE *List = fopen(File.c_str(),"r");
 209    if (List == 0)
 210       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 211
 212    /* We are a tad tricky here.. We prefix the buffer with the directory
 213       name, that way if we need a full path with just use line.. Sneaky and
 214       fully evil. */
 215    char Line[1000];
 216    char *FileStart;
 217    if (Dir.empty() == true || Dir.end()[-1] != '/')
 218       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 219    else
 220       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 221    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 222    {
 223       char *FileName = _strstrip(FileStart);
 224       if (FileName[0] == 0)
 225          continue;
 226
 227       if (FileName[0] != '/')
 228       {
 229          if (FileName != FileStart)
 230             memmove(FileStart,FileName,strlen(FileStart));
 231          FileName = Line;
 232       }
 233
 234 #if 0
 235       struct stat St;
 236       int Flag = FTW_F;
 237       if (stat(FileName,&St) != 0)
 238          Flag = FTW_NS;
 239 #endif
 240
 241       if (ScannerFile(FileName, false) != 0)
 242          break;
 243    }
 244
 245    fclose(List);
 246    return true;
 247 }
 248                                                                         /*}}}*/
 249 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 250 // ---------------------------------------------------------------------
 251 /* */
 252 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 253                         unsigned long long &DeLinkBytes,
 254                         unsigned long long const &FileSize)
 255 {
 256    // See if this isn't an internaly prefix'd file name.
 257    if (InternalPrefix.empty() == false &&
 258        InternalPrefix.length() < FileName.length() &&
 259        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 260                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 261    {
 262       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 263       {
 264          // Tidy up the display
 265          if (DeLinkBytes == 0)
 266             cout << endl;
 267
 268          NewLine(1);
 269          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 270                     SizeToStr(FileSize).c_str());
 271          c1out << flush;
 272
 273          if (NoLinkAct == false)
 274          {
 275             char OldLink[400];
 276             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 277                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 278             else
 279             {
 280                if (unlink(OriginalPath) != 0)
 281                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 282                else
 283                {
 284                   if (link(FileName.c_str(),OriginalPath) != 0)
 285                   {
 286                      // Panic! Restore the symlink
 287                      symlink(OldLink,OriginalPath);
 288                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 289                                           FileName.c_str(),
 290                                           OriginalPath);
 291                   }
 292                }
 293             }
 294          }
 295
 296          DeLinkBytes += FileSize;
 297          if (DeLinkBytes/1024 >= DeLinkLimit)
 298             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 299       }
 300
 301       FileName = OriginalPath;
 302    }
 303
 304    return true;
 305 }
 306                                                                         /*}}}*/
 307
 308 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 309 // ---------------------------------------------------------------------
 310 /* */
 311 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 312                                string const &Arch) :
 313    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 314 {
 315    Output = stdout;
 316    SetExts(".deb .udeb");
 317    DeLinkLimit = 0;
 318
 319    // Process the command line options
 320    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 321    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 322    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 323    DoSHA512 = _config->FindB("APT::FTPArchive::Packages::SHA512",DoSHA512);
 324    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 325    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 326    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 327    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 328
 329    if (Db.Loaded() == false)
 330       DoContents = false;
 331
 332    // Read the override file
 333    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 334       return;
 335    else
 336       NoOverride = true;
 337
 338    if (ExtOverrides.empty() == false)
 339       Over.ReadExtraOverride(ExtOverrides);
 340
 341    _error->DumpErrors();
 342 }
 343                                                                         /*}}}*/
 344 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 345 // ---------------------------------------------------------------------
 346 /* */
 347 bool FTWScanner::SetExts(string const &Vals)
 348 {
 349    ClearPatterns();
 350    string::size_type Start = 0;
 351    while (Start <= Vals.length()-1)
 352    {
 353       string::size_type const Space = Vals.find(' ',Start);
 354       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 355       if ( Arch.empty() == false )
 356       {
 357          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 358          AddPattern(string("*_all") + Vals.substr(Start, Length));
 359       }
 360       else
 361          AddPattern(string("*") + Vals.substr(Start, Length));
 362
 363       Start += Length + 1;
 364    }
 365
 366    return true;
 367 }
 368
 369                                                                         /*}}}*/
 370 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 371 // ---------------------------------------------------------------------
 372 /* This method takes a package and gets its control information and
 373    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 374    rewritten and the path/size/hash appended. */
 375 bool PackagesWriter::DoPackage(string FileName)
 376 {
 377    // Pull all the data we need form the DB
 378    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 379                   == false)
 380    {
 381       return false;
 382    }
 383
 384    unsigned long long FileSize = Db.GetFileSize();
 385    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 386       return false;
 387
 388    // Lookup the overide information
 389    pkgTagSection &Tags = Db.Control.Section;
 390    string Package = Tags.FindS("Package");
 391    string Architecture;
 392    // if we generate a Packages file for a given arch, we use it to
 393    // look for overrides. if we run in "simple" mode without the
 394    // "Architecures" variable in the config we use the architecure value
 395    // from the deb file
 396    if(Arch != "")
 397       Architecture = Arch;
 398    else
 399       Architecture = Tags.FindS("Architecture");
 400    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 401
 402    if (Package.empty() == true)
 403       return _error->Error(_("Archive had no package field"));
 404
 405    // If we need to do any rewriting of the header do it now..
 406    if (OverItem.get() == 0)
 407    {
 408       if (NoOverride == false)
 409       {
 410          NewLine(1);
 411          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 412       }
 413
 414       OverItem = auto_ptr<Override::Item>(new Override::Item);
 415       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 416       OverItem->Priority = Tags.FindS("Priority");
 417    }
 418
 419    char Size[40];
 420    sprintf(Size,"%llu", (unsigned long long) FileSize);
 421
 422    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 423    string NewFileName;
 424    if (DirStrip.empty() == false &&
 425        FileName.length() > DirStrip.length() &&
 426        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 427                  DirStrip.begin(),DirStrip.end()) == 0)
 428       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 429    else
 430       NewFileName = FileName;
 431    if (PathPrefix.empty() == false)
 432       NewFileName = flCombine(PathPrefix,NewFileName);
 433
 434    /* Configuration says we don't want to include the long Description
 435       in the package file - instead we want to ship a separated file */
 436    string desc;
 437    if (LongDescription == false) {
 438       desc = Tags.FindS("Description").append("\n");
 439       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 440    }
 441
 442    // This lists all the changes to the fields we are going to make.
 443    // (7 hardcoded + maintainer + suggests + end marker)
 444    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 445
 446    unsigned int End = 0;
 447    SetTFRewriteData(Changes[End++], "Size", Size);
 448    if (DoMD5 == true)
 449       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 450    if (DoSHA1 == true)
 451       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 452    if (DoSHA256 == true)
 453       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 454    if (DoSHA512 == true)
 455       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 456    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 457    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 458    SetTFRewriteData(Changes[End++], "Status", 0);
 459    SetTFRewriteData(Changes[End++], "Optional", 0);
 460
 461    string DescriptionMd5;
 462    if (LongDescription == false) {
 463       MD5Summation descmd5;
 464       descmd5.Add(desc.c_str());
 465       DescriptionMd5 = descmd5.Result().Value();
 466       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 467       if (TransWriter != NULL)
 468          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 469    }
 470
 471    // Rewrite the maintainer field if necessary
 472    bool MaintFailed;
 473    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 474    if (MaintFailed == true)
 475    {
 476       if (NoOverride == false)
 477       {
 478          NewLine(1);
 479          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 480                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 481       }
 482    }
 483
 484    if (NewMaint.empty() == false)
 485       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 486
 487    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 488       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 489       but dpkg does this append bit. So we do the append bit, at least that way the
 490       status file and package file will remain similar. There are other transforms
 491       but optional is the only legacy one still in use for some lazy reason. */
 492    string OptionalStr = Tags.FindS("Optional");
 493    if (OptionalStr.empty() == false)
 494    {
 495       if (Tags.FindS("Suggests").empty() == false)
 496          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 497       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 498    }
 499
 500    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 501         I != OverItem->FieldOverride.end(); ++I)
 502       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 503
 504    SetTFRewriteData(Changes[End++], 0, 0);
 505
 506    // Rewrite and store the fields.
 507    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 508       return false;
 509    fprintf(Output,"\n");
 510
 511    return Db.Finish();
 512 }
 513                                                                         /*}}}*/
 514
 515 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 516 // ---------------------------------------------------------------------
 517 /* Create a Translation-Master file for this Packages file */
 518 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 519                                         mode_t const &Permissions) : Output(NULL),
 520                                                         RefCounter(0)
 521 {
 522    if (File.empty() == true)
 523       return;
 524
 525    Comp = new MultiCompress(File, TransCompress, Permissions);
 526    Output = Comp->Input;
 527 }
 528                                                                         /*}}}*/
 529 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* Create a Translation-Master file for this Packages file */
 532 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 533                                   string const &MD5)
 534 {
 535    if (Output == NULL)
 536       return true;
 537
 538    // Different archs can include different versions and therefore
 539    // different descriptions - so we need to check for both name and md5.
 540    string const Record = Pkg + ":" + MD5;
 541
 542    if (Included.find(Record) != Included.end())
 543       return true;
 544
 545    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 546            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 547
 548    Included.insert(Record);
 549    return true;
 550 }
 551                                                                         /*}}}*/
 552 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* */
 555 TranslationWriter::~TranslationWriter()
 556 {
 557    if (Comp == NULL)
 558       return;
 559
 560    delete Comp;
 561 }
 562                                                                         /*}}}*/
 563
 564 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 565 // ---------------------------------------------------------------------
 566 /* */
 567 SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
 568                              string const &ExtOverrides) :
 569    Db(DB), Stats(Db.Stats)
 570 {
 571    Output = stdout;
 572    AddPattern("*.dsc");
 573    DeLinkLimit = 0;
 574    Buffer = 0;
 575    BufSize = 0;
 576
 577    // Process the command line options
 578    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 579    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 580    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 581    DoSHA512 = _config->FindB("APT::FTPArchive::Sources::SHA512",DoSHA512);
 582    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 583    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 584
 585    // Read the override file
 586    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 587       return;
 588    else
 589       NoOverride = true;
 590
 591    // WTF?? The logic above: if we can't read binary overrides, don't even try
 592    // reading source overrides. if we can read binary overrides, then say there
 593    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 594
 595    if (ExtOverrides.empty() == false)
 596       SOver.ReadExtraOverride(ExtOverrides);
 597
 598    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 599       SOver.ReadOverride(SOverrides,true);
 600 }
 601                                                                         /*}}}*/
 602 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 603 // ---------------------------------------------------------------------
 604 /* */
 605 bool SourcesWriter::DoPackage(string FileName)
 606 {
 607    // Open the archive
 608    FileFd F;
 609    if (OpenMaybeClearSignedFile(FileName, F) == false)
 610       return false;
 611
 612    unsigned long long const FSize = F.FileSize();
 613    //FIXME: do we really need to enforce a maximum size of the dsc file?
 614    if (FSize > 128*1024)
 615       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 616
 617    if (BufSize < FSize + 2)
 618    {
 619       BufSize = FSize + 2;
 620       Buffer = (char *)realloc(Buffer , BufSize);
 621    }
 622
 623    if (F.Read(Buffer, FSize) == false)
 624       return false;
 625
 626    // Stat the file for later (F might be clearsigned, so not F.FileSize())
 627    struct stat St;
 628    if (stat(FileName.c_str(), &St) != 0)
 629       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 630
 631    // Hash the file
 632    char *Start = Buffer;
 633    char *BlkEnd = Buffer + FSize;
 634
 635    Hashes DscHashes;
 636    if (FSize == (unsigned long long) St.st_size)
 637    {
 638       if (DoMD5 == true)
 639          DscHashes.MD5.Add((unsigned char *)Start,BlkEnd - Start);
 640       if (DoSHA1 == true)
 641          DscHashes.SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 642       if (DoSHA256 == true)
 643          DscHashes.SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 644       if (DoSHA512 == true)
 645          DscHashes.SHA512.Add((unsigned char *)Start,BlkEnd - Start);
 646    }
 647    else
 648    {
 649       FileFd DscFile(FileName, FileFd::ReadOnly);
 650       DscHashes.AddFD(DscFile, St.st_size, DoMD5, DoSHA1, DoSHA256, DoSHA512);
 651    }
 652
 653    // Add extra \n to the end, just in case (as in clearsigned they are missing)
 654    *BlkEnd++ = '\n';
 655    *BlkEnd++ = '\n';
 656
 657    pkgTagSection Tags;
 658    if (Tags.Scan(Start,BlkEnd - Start) == false || Tags.Exists("Source") == false)
 659       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 660    Tags.Trim();
 661
 662    // Lookup the overide information, finding first the best priority.
 663    string BestPrio;
 664    string Bins = Tags.FindS("Binary");
 665    char Buffer[Bins.length() + 1];
 666    auto_ptr<Override::Item> OverItem(0);
 667    if (Bins.empty() == false)
 668    {
 669       strcpy(Buffer,Bins.c_str());
 670
 671       // Ignore too-long errors.
 672       char *BinList[400];
 673       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 674
 675       // Look at all the binaries
 676       unsigned char BestPrioV = pkgCache::State::Extra;
 677       for (unsigned I = 0; BinList[I] != 0; I++)
 678       {
 679          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 680          if (Itm.get() == 0)
 681             continue;
 682
 683          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 684          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 685          {
 686             BestPrioV = NewPrioV;
 687             BestPrio = Itm->Priority;
 688          }
 689
 690          if (OverItem.get() == 0)
 691             OverItem = Itm;
 692       }
 693    }
 694
 695    // If we need to do any rewriting of the header do it now..
 696    if (OverItem.get() == 0)
 697    {
 698       if (NoOverride == false)
 699       {
 700          NewLine(1);
 701          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 702       }
 703
 704       OverItem = auto_ptr<Override::Item>(new Override::Item);
 705    }
 706
 707    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 708    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 709    if (SOverItem.get() == 0)
 710    {
 711       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 712       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 713       if (SOverItem.get() == 0)
 714       {
 715         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 716          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 717          *SOverItem = *OverItem;
 718       }
 719    }
 720
 721    // Add the dsc to the files hash list
 722    string const strippedName = flNotDir(FileName);
 723    std::ostringstream ostreamFiles;
 724    if (DoMD5 == true && Tags.Exists("Files"))
 725       ostreamFiles << "\n " << string(DscHashes.MD5.Result()) << " " << St.st_size << " "
 726                    << strippedName << "\n " << Tags.FindS("Files");
 727    string const Files = ostreamFiles.str();
 728
 729    std::ostringstream ostreamSha1;
 730    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 731       ostreamSha1 << "\n " << string(DscHashes.SHA1.Result()) << " " << St.st_size << " "
 732                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 733
 734    std::ostringstream ostreamSha256;
 735    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 736       ostreamSha256 << "\n " << string(DscHashes.SHA256.Result()) << " " << St.st_size << " "
 737                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 738
 739    std::ostringstream ostreamSha512;
 740    if (DoSHA512 == true && Tags.Exists("Checksums-Sha512"))
 741       ostreamSha512 << "\n " << string(DscHashes.SHA512.Result()) << " " << St.st_size << " "
 742                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 743
 744    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 745    string NewFileName;
 746    if (DirStrip.empty() == false &&
 747        FileName.length() > DirStrip.length() &&
 748        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 749       NewFileName = string(OriginalPath + DirStrip.length());
 750    else
 751       NewFileName = OriginalPath;
 752    if (PathPrefix.empty() == false)
 753       NewFileName = flCombine(PathPrefix,NewFileName);
 754
 755    string Directory = flNotFile(OriginalPath);
 756    string Package = Tags.FindS("Source");
 757
 758    // Perform operation over all of the files
 759    string ParseJnk;
 760    const char *C = Files.c_str();
 761    char *RealPath = NULL;
 762    for (;isspace(*C); C++);
 763    while (*C != 0)
 764    {
 765       // Parse each of the elements
 766       if (ParseQuoteWord(C,ParseJnk) == false ||
 767           ParseQuoteWord(C,ParseJnk) == false ||
 768           ParseQuoteWord(C,ParseJnk) == false)
 769          return _error->Error("Error parsing file record");
 770
 771       string OriginalPath = Directory + ParseJnk;
 772
 773       // Add missing hashes to source files
 774       if ((DoSHA1 == true && !Tags.Exists("Checksums-Sha1")) ||
 775           (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) ||
 776           (DoSHA512 == true && !Tags.Exists("Checksums-Sha512")))
 777       {
 778          if (Db.GetFileInfo(OriginalPath, false, false, false, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 779                == false)
 780          {
 781             return _error->Error("Error getting file info");
 782          }
 783
 784          if (DoSHA1 == true && !Tags.Exists("Checksums-Sha1"))
 785             ostreamSha1 << "\n " << string(Db.SHA1Res) << " "
 786                << Db.GetFileSize() << " " << ParseJnk;
 787
 788          if (DoSHA256 == true && !Tags.Exists("Checksums-Sha256"))
 789             ostreamSha256 << "\n " << string(Db.SHA256Res) << " "
 790                << Db.GetFileSize() << " " << ParseJnk;
 791
 792          if (DoSHA512 == true && !Tags.Exists("Checksums-Sha512"))
 793             ostreamSha512 << "\n " << string(Db.SHA512Res) << " "
 794                << Db.GetFileSize() << " " << ParseJnk;
 795       }
 796
 797       // Perform the delinking operation
 798       char Jnk[2];
 799
 800       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 801           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 802       {
 803          string RP = RealPath;
 804          free(RealPath);
 805          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 806             return false;
 807       }
 808    }
 809
 810    Directory = flNotFile(NewFileName);
 811    if (Directory.length() > 2)
 812       Directory.erase(Directory.end()-1);
 813
 814    string const ChecksumsSha1 = ostreamSha1.str();
 815    string const ChecksumsSha256 = ostreamSha256.str();
 816    string const ChecksumsSha512 = ostreamSha512.str();
 817
 818    // This lists all the changes to the fields we are going to make.
 819    // (5 hardcoded + checksums + maintainer + end marker)
 820    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 821
 822    unsigned int End = 0;
 823    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 824    if (Files.empty() == false)
 825       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 826    if (ChecksumsSha1.empty() == false)
 827       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 828    if (ChecksumsSha256.empty() == false)
 829       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 830    if (ChecksumsSha512.empty() == false)
 831       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 832    if (Directory != "./")
 833       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 834    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 835    SetTFRewriteData(Changes[End++],"Status",0);
 836
 837    // Rewrite the maintainer field if necessary
 838    bool MaintFailed;
 839    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 840    if (MaintFailed == true)
 841    {
 842       if (NoOverride == false)
 843       {
 844          NewLine(1);
 845          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 846                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 847       }
 848    }
 849    if (NewMaint.empty() == false)
 850       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 851
 852    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 853         I != SOverItem->FieldOverride.end(); ++I)
 854       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 855
 856    SetTFRewriteData(Changes[End++], 0, 0);
 857
 858    // Rewrite and store the fields.
 859    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 860       return false;
 861    fprintf(Output,"\n");
 862
 863    Stats.Packages++;
 864
 865    return Db.Finish();
 866 }
 867                                                                         /*}}}*/
 868
 869 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 870 // ---------------------------------------------------------------------
 871 /* */
 872 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 873                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 874
 875 {
 876    SetExts(".deb");
 877    Output = stdout;
 878 }
 879                                                                         /*}}}*/
 880 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 881 // ---------------------------------------------------------------------
 882 /* If Package is the empty string the control record will be parsed to
 883    determine what the package name is. */
 884 bool ContentsWriter::DoPackage(string FileName, string Package)
 885 {
 886    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 887    {
 888       return false;
 889    }
 890
 891    // Parse the package name
 892    if (Package.empty() == true)
 893    {
 894       Package = Db.Control.Section.FindS("Package");
 895    }
 896
 897    Db.Contents.Add(Gen,Package);
 898
 899    return Db.Finish();
 900 }
 901                                                                         /*}}}*/
 902 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 903 // ---------------------------------------------------------------------
 904 /* */
 905 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 906 {
 907    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 908    if (_error->PendingError() == true)
 909       return false;
 910
 911    // Open the package file
 912    FileFd Fd;
 913    if (Pkgs.OpenOld(Fd) == false)
 914       return false;
 915
 916    pkgTagFile Tags(&Fd);
 917    if (_error->PendingError() == true)
 918       return false;
 919
 920    // Parse.
 921    pkgTagSection Section;
 922    while (Tags.Step(Section) == true)
 923    {
 924       string File = flCombine(Prefix,Section.FindS("FileName"));
 925       string Package = Section.FindS("Section");
 926       if (Package.empty() == false && Package.end()[-1] != '/')
 927       {
 928          Package += '/';
 929          Package += Section.FindS("Package");
 930       }
 931       else
 932          Package += Section.FindS("Package");
 933
 934       DoPackage(File,Package);
 935       if (_error->empty() == false)
 936       {
 937          _error->Error("Errors apply to file '%s'",File.c_str());
 938          _error->DumpErrors();
 939       }
 940    }
 941
 942    // Tidy the compressor
 943    Fd.Close();
 944
 945    return true;
 946 }
 947
 948                                                                         /*}}}*/
 949
 950 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 951 // ---------------------------------------------------------------------
 952 /* */
 953 ReleaseWriter::ReleaseWriter(string const &DB)
 954 {
 955    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 956    {
 957       AddPattern("Packages");
 958       AddPattern("Packages.gz");
 959       AddPattern("Packages.bz2");
 960       AddPattern("Packages.lzma");
 961       AddPattern("Packages.xz");
 962       AddPattern("Translation-*");
 963       AddPattern("Sources");
 964       AddPattern("Sources.gz");
 965       AddPattern("Sources.bz2");
 966       AddPattern("Sources.lzma");
 967       AddPattern("Sources.xz");
 968       AddPattern("Release");
 969       AddPattern("Contents-*");
 970       AddPattern("Index");
 971       AddPattern("md5sum.txt");
 972    }
 973    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 974
 975    Output = stdout;
 976    time_t const now = time(NULL);
 977
 978    setlocale(LC_TIME, "C");
 979
 980    char datestr[128];
 981    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 982                 gmtime(&now)) == 0)
 983    {
 984       datestr[0] = '\0';
 985    }
 986
 987    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 988    char validstr[128];
 989    if (now == validuntil ||
 990        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 991                 gmtime(&validuntil)) == 0)
 992    {
 993       validstr[0] = '\0';
 994    }
 995
 996    setlocale(LC_TIME, "");
 997
 998    map<string,string> Fields;
 999    Fields["Origin"] = "";
1000    Fields["Label"] = "";
1001    Fields["Suite"] = "";
1002    Fields["Version"] = "";
1003    Fields["Codename"] = "";
1004    Fields["Date"] = datestr;
1005    Fields["Valid-Until"] = validstr;
1006    Fields["Architectures"] = "";
1007    Fields["Components"] = "";
1008    Fields["Description"] = "";
1009
1010    for(map<string,string>::const_iterator I = Fields.begin();
1011        I != Fields.end();
1012        ++I)
1013    {
1014       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1015       string Value = _config->Find(Config, (*I).second.c_str());
1016       if (Value == "")
1017          continue;
1018
1019       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1020    }
1021
1022    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1023    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1024    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1025 }
1026                                                                         /*}}}*/
1027 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1028 // ---------------------------------------------------------------------
1029 bool ReleaseWriter::DoPackage(string FileName)
1030 {
1031    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1032    string NewFileName;
1033    if (DirStrip.empty() == false &&
1034        FileName.length() > DirStrip.length() &&
1035        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1036                  DirStrip.begin(),DirStrip.end()) == 0)
1037    {
1038       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1039       while (NewFileName[0] == '/')
1040          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1041    }
1042    else
1043       NewFileName = FileName;
1044
1045    if (PathPrefix.empty() == false)
1046       NewFileName = flCombine(PathPrefix,NewFileName);
1047
1048    FileFd fd(FileName, FileFd::ReadOnly);
1049
1050    if (!fd.IsOpen())
1051    {
1052       return false;
1053    }
1054
1055    CheckSums[NewFileName].size = fd.Size();
1056
1057    Hashes hs;
1058    hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1059    if (DoMD5 == true)
1060       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1061    if (DoSHA1 == true)
1062       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1063    if (DoSHA256 == true)
1064       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1065    if (DoSHA512 == true)
1066       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1067    fd.Close();
1068
1069    return true;
1070 }
1071
1072                                                                         /*}}}*/
1073 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1074 // ---------------------------------------------------------------------
1075 void ReleaseWriter::Finish()
1076 {
1077    if (DoMD5 == true)
1078    {
1079       fprintf(Output, "MD5Sum:\n");
1080       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1081           I != CheckSums.end(); ++I)
1082       {
1083          fprintf(Output, " %s %16llu %s\n",
1084                  (*I).second.MD5.c_str(),
1085                  (*I).second.size,
1086                  (*I).first.c_str());
1087       }
1088    }
1089    if (DoSHA1 == true)
1090    {
1091       fprintf(Output, "SHA1:\n");
1092       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1093           I != CheckSums.end(); ++I)
1094       {
1095          fprintf(Output, " %s %16llu %s\n",
1096                  (*I).second.SHA1.c_str(),
1097                  (*I).second.size,
1098                  (*I).first.c_str());
1099       }
1100    }
1101    if (DoSHA256 == true)
1102    {
1103       fprintf(Output, "SHA256:\n");
1104       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1105           I != CheckSums.end(); ++I)
1106       {
1107          fprintf(Output, " %s %16llu %s\n",
1108                  (*I).second.SHA256.c_str(),
1109                  (*I).second.size,
1110                  (*I).first.c_str());
1111       }
1112    }
1113
1114    fprintf(Output, "SHA512:\n");
1115    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1116        I != CheckSums.end();
1117        ++I)
1118    {
1119       fprintf(Output, " %s %16llu %s\n",
1120               (*I).second.SHA512.c_str(),
1121               (*I).second.size,
1122               (*I).first.c_str());
1123    }
1124
1125 }