ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/error.h>
  18 #include <apt-pkg/configuration.h>
  19 #include <apt-pkg/aptconfiguration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/deblistparser.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/gpgv.h>
  25
  26 #include <sys/types.h>
  27 #include <unistd.h>
  28 #include <ctime>
  29 #include <ftw.h>
  30 #include <fnmatch.h>
  31 #include <iostream>
  32 #include <sstream>
  33 #include <memory>
  34
  35 #include "writer.h"
  36 #include "cachedb.h"
  37 #include "apt-ftparchive.h"
  38 #include "multicompress.h"
  39
  40 #include <apti18n.h>
  41                                                                         /*}}}*/
  42 using namespace std;
  43 FTWScanner *FTWScanner::Owner;
  44
  45 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  46 // ---------------------------------------------------------------------
  47 /* */
  48 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  49                              const char *tag,
  50                              const char *rewrite,
  51                              const char *newtag = 0)
  52 {
  53     tfrd.Tag = tag;
  54     tfrd.Rewrite = rewrite;
  55     tfrd.NewTag = newtag;
  56 }
  57                                                                         /*}}}*/
  58
  59 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  63 {
  64    ErrorPrinted = false;
  65    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  66
  67    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  68    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  69    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  70    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  71 }
  72                                                                         /*}}}*/
  73 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  74 // ---------------------------------------------------------------------
  75 /* This is the FTW scanner, it processes each directory element in the
  76    directory tree. */
  77 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  78 {
  79    if (Flag == FTW_DNR)
  80    {
  81       Owner->NewLine(1);
  82       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  83    }
  84    if (Flag == FTW_NS)
  85    {
  86       Owner->NewLine(1);
  87       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  88    }
  89    if (Flag != FTW_F)
  90       return 0;
  91
  92    return ScannerFile(File, true);
  93 }
  94                                                                         /*}}}*/
  95 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* */
  98 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  99 {
 100    const char *LastComponent = strrchr(File, '/');
 101    char *RealPath = NULL;
 102
 103    if (LastComponent == NULL)
 104       LastComponent = File;
 105    else
 106       LastComponent++;
 107
 108    vector<string>::const_iterator I;
 109    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 110    {
 111       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 112          break;
 113    }
 114    if (I == Owner->Patterns.end())
 115       return 0;
 116
 117    /* Process it. If the file is a link then resolve it into an absolute
 118       name.. This works best if the directory components the scanner are
 119       given are not links themselves. */
 120    char Jnk[2];
 121    Owner->OriginalPath = File;
 122    if (ReadLink &&
 123        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 124        (RealPath = realpath(File,NULL)) != 0)
 125    {
 126       Owner->DoPackage(RealPath);
 127       free(RealPath);
 128    }
 129    else
 130       Owner->DoPackage(File);
 131
 132    if (_error->empty() == false)
 133    {
 134       // Print any errors or warnings found
 135       string Err;
 136       bool SeenPath = false;
 137       while (_error->empty() == false)
 138       {
 139          Owner->NewLine(1);
 140
 141          bool const Type = _error->PopMessage(Err);
 142          if (Type == true)
 143             cerr << _("E: ") << Err << endl;
 144          else
 145             cerr << _("W: ") << Err << endl;
 146
 147          if (Err.find(File) != string::npos)
 148             SeenPath = true;
 149       }
 150
 151       if (SeenPath == false)
 152          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 153       return 0;
 154    }
 155
 156    return 0;
 157 }
 158                                                                         /*}}}*/
 159 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 160 // ---------------------------------------------------------------------
 161 /* */
 162 bool FTWScanner::RecursiveScan(string const &Dir)
 163 {
 164    char *RealPath = NULL;
 165    /* If noprefix is set then jam the scan root in, so we don't generate
 166       link followed paths out of control */
 167    if (InternalPrefix.empty() == true)
 168    {
 169       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 170          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 171       InternalPrefix = RealPath;
 172       free(RealPath);
 173    }
 174
 175    // Do recursive directory searching
 176    Owner = this;
 177    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 178
 179    // Error treewalking?
 180    if (Res != 0)
 181    {
 182       if (_error->PendingError() == false)
 183          _error->Errno("ftw",_("Tree walking failed"));
 184       return false;
 185    }
 186
 187    return true;
 188 }
 189                                                                         /*}}}*/
 190 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 191 // ---------------------------------------------------------------------
 192 /* This is an alternative to using FTW to locate files, it reads the list
 193    of files from another file. */
 194 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 195 {
 196    char *RealPath = NULL;
 197    /* If noprefix is set then jam the scan root in, so we don't generate
 198       link followed paths out of control */
 199    if (InternalPrefix.empty() == true)
 200    {
 201       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 202          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 203       InternalPrefix = RealPath;
 204       free(RealPath);
 205    }
 206
 207    Owner = this;
 208    FILE *List = fopen(File.c_str(),"r");
 209    if (List == 0)
 210       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 211
 212    /* We are a tad tricky here.. We prefix the buffer with the directory
 213       name, that way if we need a full path with just use line.. Sneaky and
 214       fully evil. */
 215    char Line[1000];
 216    char *FileStart;
 217    if (Dir.empty() == true || Dir.end()[-1] != '/')
 218       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 219    else
 220       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 221    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 222    {
 223       char *FileName = _strstrip(FileStart);
 224       if (FileName[0] == 0)
 225          continue;
 226
 227       if (FileName[0] != '/')
 228       {
 229          if (FileName != FileStart)
 230             memmove(FileStart,FileName,strlen(FileStart));
 231          FileName = Line;
 232       }
 233
 234 #if 0
 235       struct stat St;
 236       int Flag = FTW_F;
 237       if (stat(FileName,&St) != 0)
 238          Flag = FTW_NS;
 239 #endif
 240
 241       if (ScannerFile(FileName, false) != 0)
 242          break;
 243    }
 244
 245    fclose(List);
 246    return true;
 247 }
 248                                                                         /*}}}*/
 249 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 250 // ---------------------------------------------------------------------
 251 /* */
 252 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 253                         unsigned long long &DeLinkBytes,
 254                         unsigned long long const &FileSize)
 255 {
 256    // See if this isn't an internaly prefix'd file name.
 257    if (InternalPrefix.empty() == false &&
 258        InternalPrefix.length() < FileName.length() &&
 259        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 260                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 261    {
 262       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 263       {
 264          // Tidy up the display
 265          if (DeLinkBytes == 0)
 266             cout << endl;
 267
 268          NewLine(1);
 269          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 270                     SizeToStr(FileSize).c_str());
 271          c1out << flush;
 272
 273          if (NoLinkAct == false)
 274          {
 275             char OldLink[400];
 276             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 277                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 278             else
 279             {
 280                if (unlink(OriginalPath) != 0)
 281                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 282                else
 283                {
 284                   if (link(FileName.c_str(),OriginalPath) != 0)
 285                   {
 286                      // Panic! Restore the symlink
 287                      if (symlink(OldLink,OriginalPath) != 0)
 288                         _error->Errno("symlink", "failed to restore symlink");
 289                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 290                                           FileName.c_str(),
 291                                           OriginalPath);
 292                   }
 293                }
 294             }
 295          }
 296
 297          DeLinkBytes += FileSize;
 298          if (DeLinkBytes/1024 >= DeLinkLimit)
 299             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 300       }
 301
 302       FileName = OriginalPath;
 303    }
 304
 305    return true;
 306 }
 307                                                                         /*}}}*/
 308
 309 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 310 // ---------------------------------------------------------------------
 311 /* */
 312 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 313                                string const &Arch) :
 314    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 315 {
 316    Output = stdout;
 317    SetExts(".deb .udeb");
 318    DeLinkLimit = 0;
 319
 320    // Process the command line options
 321    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 322    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 323    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 324    DoSHA512 = _config->FindB("APT::FTPArchive::Packages::SHA512",DoSHA512);
 325    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 326    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 327    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 328    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 329
 330    if (Db.Loaded() == false)
 331       DoContents = false;
 332
 333    // Read the override file
 334    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 335       return;
 336    else
 337       NoOverride = true;
 338
 339    if (ExtOverrides.empty() == false)
 340       Over.ReadExtraOverride(ExtOverrides);
 341
 342    _error->DumpErrors();
 343 }
 344                                                                         /*}}}*/
 345 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 346 // ---------------------------------------------------------------------
 347 /* */
 348 bool FTWScanner::SetExts(string const &Vals)
 349 {
 350    ClearPatterns();
 351    string::size_type Start = 0;
 352    while (Start <= Vals.length()-1)
 353    {
 354       string::size_type const Space = Vals.find(' ',Start);
 355       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 356       if ( Arch.empty() == false )
 357       {
 358          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 359          AddPattern(string("*_all") + Vals.substr(Start, Length));
 360       }
 361       else
 362          AddPattern(string("*") + Vals.substr(Start, Length));
 363
 364       Start += Length + 1;
 365    }
 366
 367    return true;
 368 }
 369
 370                                                                         /*}}}*/
 371 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 372 // ---------------------------------------------------------------------
 373 /* This method takes a package and gets its control information and
 374    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 375    rewritten and the path/size/hash appended. */
 376 bool PackagesWriter::DoPackage(string FileName)
 377 {
 378    // Pull all the data we need form the DB
 379    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 380                   == false)
 381    {
 382       return false;
 383    }
 384
 385    unsigned long long FileSize = Db.GetFileSize();
 386    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 387       return false;
 388
 389    // Lookup the overide information
 390    pkgTagSection &Tags = Db.Control.Section;
 391    string Package = Tags.FindS("Package");
 392    string Architecture;
 393    // if we generate a Packages file for a given arch, we use it to
 394    // look for overrides. if we run in "simple" mode without the
 395    // "Architecures" variable in the config we use the architecure value
 396    // from the deb file
 397    if(Arch != "")
 398       Architecture = Arch;
 399    else
 400       Architecture = Tags.FindS("Architecture");
 401    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 402
 403    if (Package.empty() == true)
 404       return _error->Error(_("Archive had no package field"));
 405
 406    // If we need to do any rewriting of the header do it now..
 407    if (OverItem.get() == 0)
 408    {
 409       if (NoOverride == false)
 410       {
 411          NewLine(1);
 412          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 413       }
 414
 415       OverItem = auto_ptr<Override::Item>(new Override::Item);
 416       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 417       OverItem->Priority = Tags.FindS("Priority");
 418    }
 419
 420    char Size[40];
 421    sprintf(Size,"%llu", (unsigned long long) FileSize);
 422
 423    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 424    string NewFileName;
 425    if (DirStrip.empty() == false &&
 426        FileName.length() > DirStrip.length() &&
 427        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 428                  DirStrip.begin(),DirStrip.end()) == 0)
 429       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 430    else
 431       NewFileName = FileName;
 432    if (PathPrefix.empty() == false)
 433       NewFileName = flCombine(PathPrefix,NewFileName);
 434
 435    /* Configuration says we don't want to include the long Description
 436       in the package file - instead we want to ship a separated file */
 437    string desc;
 438    if (LongDescription == false) {
 439       desc = Tags.FindS("Description").append("\n");
 440       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 441    }
 442
 443    // This lists all the changes to the fields we are going to make.
 444    // (7 hardcoded + maintainer + suggests + end marker)
 445    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 446
 447    unsigned int End = 0;
 448    SetTFRewriteData(Changes[End++], "Size", Size);
 449    if (DoMD5 == true)
 450       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 451    if (DoSHA1 == true)
 452       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 453    if (DoSHA256 == true)
 454       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 455    if (DoSHA512 == true)
 456       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 457    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 458    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 459    SetTFRewriteData(Changes[End++], "Status", 0);
 460    SetTFRewriteData(Changes[End++], "Optional", 0);
 461
 462    string DescriptionMd5;
 463    if (LongDescription == false) {
 464       MD5Summation descmd5;
 465       descmd5.Add(desc.c_str());
 466       DescriptionMd5 = descmd5.Result().Value();
 467       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 468       if (TransWriter != NULL)
 469          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 470    }
 471
 472    // Rewrite the maintainer field if necessary
 473    bool MaintFailed;
 474    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 475    if (MaintFailed == true)
 476    {
 477       if (NoOverride == false)
 478       {
 479          NewLine(1);
 480          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 481                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 482       }
 483    }
 484
 485    if (NewMaint.empty() == false)
 486       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 487
 488    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 489       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 490       but dpkg does this append bit. So we do the append bit, at least that way the
 491       status file and package file will remain similar. There are other transforms
 492       but optional is the only legacy one still in use for some lazy reason. */
 493    string OptionalStr = Tags.FindS("Optional");
 494    if (OptionalStr.empty() == false)
 495    {
 496       if (Tags.FindS("Suggests").empty() == false)
 497          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 498       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 499    }
 500
 501    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 502         I != OverItem->FieldOverride.end(); ++I)
 503       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 504
 505    SetTFRewriteData(Changes[End++], 0, 0);
 506
 507    // Rewrite and store the fields.
 508    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 509       return false;
 510    fprintf(Output,"\n");
 511
 512    return Db.Finish();
 513 }
 514                                                                         /*}}}*/
 515
 516 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 517 // ---------------------------------------------------------------------
 518 /* Create a Translation-Master file for this Packages file */
 519 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 520                                         mode_t const &Permissions) : Output(NULL),
 521                                                         RefCounter(0)
 522 {
 523    if (File.empty() == true)
 524       return;
 525
 526    Comp = new MultiCompress(File, TransCompress, Permissions);
 527    Output = Comp->Input;
 528 }
 529                                                                         /*}}}*/
 530 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 531 // ---------------------------------------------------------------------
 532 /* Create a Translation-Master file for this Packages file */
 533 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 534                                   string const &MD5)
 535 {
 536    if (Output == NULL)
 537       return true;
 538
 539    // Different archs can include different versions and therefore
 540    // different descriptions - so we need to check for both name and md5.
 541    string const Record = Pkg + ":" + MD5;
 542
 543    if (Included.find(Record) != Included.end())
 544       return true;
 545
 546    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 547            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 548
 549    Included.insert(Record);
 550    return true;
 551 }
 552                                                                         /*}}}*/
 553 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 554 // ---------------------------------------------------------------------
 555 /* */
 556 TranslationWriter::~TranslationWriter()
 557 {
 558    if (Comp == NULL)
 559       return;
 560
 561    delete Comp;
 562 }
 563                                                                         /*}}}*/
 564
 565 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 566 // ---------------------------------------------------------------------
 567 /* */
 568 SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
 569                              string const &ExtOverrides) :
 570    Db(DB), Stats(Db.Stats)
 571 {
 572    Output = stdout;
 573    AddPattern("*.dsc");
 574    DeLinkLimit = 0;
 575    Buffer = 0;
 576    BufSize = 0;
 577
 578    // Process the command line options
 579    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 580    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 581    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 582    DoSHA512 = _config->FindB("APT::FTPArchive::Sources::SHA512",DoSHA512);
 583    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 584    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 585
 586    // Read the override file
 587    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 588       return;
 589    else
 590       NoOverride = true;
 591
 592    // WTF?? The logic above: if we can't read binary overrides, don't even try
 593    // reading source overrides. if we can read binary overrides, then say there
 594    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 595
 596    if (ExtOverrides.empty() == false)
 597       SOver.ReadExtraOverride(ExtOverrides);
 598
 599    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 600       SOver.ReadOverride(SOverrides,true);
 601 }
 602                                                                         /*}}}*/
 603 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 604 // ---------------------------------------------------------------------
 605 /* */
 606 bool SourcesWriter::DoPackage(string FileName)
 607 {
 608    // Open the archive
 609    FileFd F;
 610    if (OpenMaybeClearSignedFile(FileName, F) == false)
 611       return false;
 612
 613    unsigned long long const FSize = F.FileSize();
 614    //FIXME: do we really need to enforce a maximum size of the dsc file?
 615    if (FSize > 128*1024)
 616       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 617
 618    if (BufSize < FSize + 2)
 619    {
 620       BufSize = FSize + 2;
 621       Buffer = (char *)realloc(Buffer , BufSize);
 622    }
 623
 624    if (F.Read(Buffer, FSize) == false)
 625       return false;
 626
 627    // Stat the file for later (F might be clearsigned, so not F.FileSize())
 628    struct stat St;
 629    if (stat(FileName.c_str(), &St) != 0)
 630       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 631
 632    // Hash the file
 633    char *Start = Buffer;
 634    char *BlkEnd = Buffer + FSize;
 635
 636    Hashes DscHashes;
 637    if (FSize == (unsigned long long) St.st_size)
 638    {
 639       if (DoMD5 == true)
 640          DscHashes.MD5.Add((unsigned char *)Start,BlkEnd - Start);
 641       if (DoSHA1 == true)
 642          DscHashes.SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 643       if (DoSHA256 == true)
 644          DscHashes.SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 645       if (DoSHA512 == true)
 646          DscHashes.SHA512.Add((unsigned char *)Start,BlkEnd - Start);
 647    }
 648    else
 649    {
 650       FileFd DscFile(FileName, FileFd::ReadOnly);
 651       DscHashes.AddFD(DscFile, St.st_size, DoMD5, DoSHA1, DoSHA256, DoSHA512);
 652    }
 653
 654    // Add extra \n to the end, just in case (as in clearsigned they are missing)
 655    *BlkEnd++ = '\n';
 656    *BlkEnd++ = '\n';
 657
 658    pkgTagSection Tags;
 659    if (Tags.Scan(Start,BlkEnd - Start) == false || Tags.Exists("Source") == false)
 660       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 661    Tags.Trim();
 662
 663    // Lookup the overide information, finding first the best priority.
 664    string BestPrio;
 665    string Bins = Tags.FindS("Binary");
 666    char Buffer[Bins.length() + 1];
 667    auto_ptr<Override::Item> OverItem(0);
 668    if (Bins.empty() == false)
 669    {
 670       strcpy(Buffer,Bins.c_str());
 671
 672       // Ignore too-long errors.
 673       char *BinList[400];
 674       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 675
 676       // Look at all the binaries
 677       unsigned char BestPrioV = pkgCache::State::Extra;
 678       for (unsigned I = 0; BinList[I] != 0; I++)
 679       {
 680          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 681          if (Itm.get() == 0)
 682             continue;
 683
 684          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 685          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 686          {
 687             BestPrioV = NewPrioV;
 688             BestPrio = Itm->Priority;
 689          }
 690
 691          if (OverItem.get() == 0)
 692             OverItem = Itm;
 693       }
 694    }
 695
 696    // If we need to do any rewriting of the header do it now..
 697    if (OverItem.get() == 0)
 698    {
 699       if (NoOverride == false)
 700       {
 701          NewLine(1);
 702          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 703       }
 704
 705       OverItem = auto_ptr<Override::Item>(new Override::Item);
 706    }
 707
 708    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 709    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 710    if (SOverItem.get() == 0)
 711    {
 712       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 713       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 714       if (SOverItem.get() == 0)
 715       {
 716         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 717          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 718          *SOverItem = *OverItem;
 719       }
 720    }
 721
 722    // Add the dsc to the files hash list
 723    string const strippedName = flNotDir(FileName);
 724    std::ostringstream ostreamFiles;
 725    if (DoMD5 == true && Tags.Exists("Files"))
 726       ostreamFiles << "\n " << string(DscHashes.MD5.Result()) << " " << St.st_size << " "
 727                    << strippedName << "\n " << Tags.FindS("Files");
 728    string const Files = ostreamFiles.str();
 729
 730    std::ostringstream ostreamSha1;
 731    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 732       ostreamSha1 << "\n " << string(DscHashes.SHA1.Result()) << " " << St.st_size << " "
 733                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 734
 735    std::ostringstream ostreamSha256;
 736    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 737       ostreamSha256 << "\n " << string(DscHashes.SHA256.Result()) << " " << St.st_size << " "
 738                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 739
 740    std::ostringstream ostreamSha512;
 741    if (DoSHA512 == true && Tags.Exists("Checksums-Sha512"))
 742       ostreamSha512 << "\n " << string(DscHashes.SHA512.Result()) << " " << St.st_size << " "
 743                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 744
 745    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 746    string NewFileName;
 747    if (DirStrip.empty() == false &&
 748        FileName.length() > DirStrip.length() &&
 749        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 750       NewFileName = string(OriginalPath + DirStrip.length());
 751    else
 752       NewFileName = OriginalPath;
 753    if (PathPrefix.empty() == false)
 754       NewFileName = flCombine(PathPrefix,NewFileName);
 755
 756    string Directory = flNotFile(OriginalPath);
 757    string Package = Tags.FindS("Source");
 758
 759    // Perform operation over all of the files
 760    string ParseJnk;
 761    const char *C = Files.c_str();
 762    char *RealPath = NULL;
 763    for (;isspace(*C); C++);
 764    while (*C != 0)
 765    {
 766       // Parse each of the elements
 767       if (ParseQuoteWord(C,ParseJnk) == false ||
 768           ParseQuoteWord(C,ParseJnk) == false ||
 769           ParseQuoteWord(C,ParseJnk) == false)
 770          return _error->Error("Error parsing file record");
 771
 772       string OriginalPath = Directory + ParseJnk;
 773
 774       // Add missing hashes to source files
 775       if ((DoSHA1 == true && !Tags.Exists("Checksums-Sha1")) ||
 776           (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) ||
 777           (DoSHA512 == true && !Tags.Exists("Checksums-Sha512")))
 778       {
 779          if (Db.GetFileInfo(OriginalPath, false, false, false, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 780                == false)
 781          {
 782             return _error->Error("Error getting file info");
 783          }
 784
 785          if (DoSHA1 == true && !Tags.Exists("Checksums-Sha1"))
 786             ostreamSha1 << "\n " << string(Db.SHA1Res) << " "
 787                << Db.GetFileSize() << " " << ParseJnk;
 788
 789          if (DoSHA256 == true && !Tags.Exists("Checksums-Sha256"))
 790             ostreamSha256 << "\n " << string(Db.SHA256Res) << " "
 791                << Db.GetFileSize() << " " << ParseJnk;
 792
 793          if (DoSHA512 == true && !Tags.Exists("Checksums-Sha512"))
 794             ostreamSha512 << "\n " << string(Db.SHA512Res) << " "
 795                << Db.GetFileSize() << " " << ParseJnk;
 796       }
 797
 798       // Perform the delinking operation
 799       char Jnk[2];
 800
 801       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 802           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 803       {
 804          string RP = RealPath;
 805          free(RealPath);
 806          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 807             return false;
 808       }
 809    }
 810
 811    Directory = flNotFile(NewFileName);
 812    if (Directory.length() > 2)
 813       Directory.erase(Directory.end()-1);
 814
 815    string const ChecksumsSha1 = ostreamSha1.str();
 816    string const ChecksumsSha256 = ostreamSha256.str();
 817    string const ChecksumsSha512 = ostreamSha512.str();
 818
 819    // This lists all the changes to the fields we are going to make.
 820    // (5 hardcoded + checksums + maintainer + end marker)
 821    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 822
 823    unsigned int End = 0;
 824    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 825    if (Files.empty() == false)
 826       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 827    if (ChecksumsSha1.empty() == false)
 828       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 829    if (ChecksumsSha256.empty() == false)
 830       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 831    if (ChecksumsSha512.empty() == false)
 832       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 833    if (Directory != "./")
 834       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 835    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 836    SetTFRewriteData(Changes[End++],"Status",0);
 837
 838    // Rewrite the maintainer field if necessary
 839    bool MaintFailed;
 840    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 841    if (MaintFailed == true)
 842    {
 843       if (NoOverride == false)
 844       {
 845          NewLine(1);
 846          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 847                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 848       }
 849    }
 850    if (NewMaint.empty() == false)
 851       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 852
 853    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 854         I != SOverItem->FieldOverride.end(); ++I)
 855       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 856
 857    SetTFRewriteData(Changes[End++], 0, 0);
 858
 859    // Rewrite and store the fields.
 860    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 861       return false;
 862    fprintf(Output,"\n");
 863
 864    Stats.Packages++;
 865
 866    return Db.Finish();
 867 }
 868                                                                         /*}}}*/
 869
 870 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 871 // ---------------------------------------------------------------------
 872 /* */
 873 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 874                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 875
 876 {
 877    SetExts(".deb");
 878    Output = stdout;
 879 }
 880                                                                         /*}}}*/
 881 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 882 // ---------------------------------------------------------------------
 883 /* If Package is the empty string the control record will be parsed to
 884    determine what the package name is. */
 885 bool ContentsWriter::DoPackage(string FileName, string Package)
 886 {
 887    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 888    {
 889       return false;
 890    }
 891
 892    // Parse the package name
 893    if (Package.empty() == true)
 894    {
 895       Package = Db.Control.Section.FindS("Package");
 896    }
 897
 898    Db.Contents.Add(Gen,Package);
 899
 900    return Db.Finish();
 901 }
 902                                                                         /*}}}*/
 903 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 904 // ---------------------------------------------------------------------
 905 /* */
 906 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 907 {
 908    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 909    if (_error->PendingError() == true)
 910       return false;
 911
 912    // Open the package file
 913    FileFd Fd;
 914    if (Pkgs.OpenOld(Fd) == false)
 915       return false;
 916
 917    pkgTagFile Tags(&Fd);
 918    if (_error->PendingError() == true)
 919       return false;
 920
 921    // Parse.
 922    pkgTagSection Section;
 923    while (Tags.Step(Section) == true)
 924    {
 925       string File = flCombine(Prefix,Section.FindS("FileName"));
 926       string Package = Section.FindS("Section");
 927       if (Package.empty() == false && Package.end()[-1] != '/')
 928       {
 929          Package += '/';
 930          Package += Section.FindS("Package");
 931       }
 932       else
 933          Package += Section.FindS("Package");
 934
 935       DoPackage(File,Package);
 936       if (_error->empty() == false)
 937       {
 938          _error->Error("Errors apply to file '%s'",File.c_str());
 939          _error->DumpErrors();
 940       }
 941    }
 942
 943    // Tidy the compressor
 944    Fd.Close();
 945
 946    return true;
 947 }
 948
 949                                                                         /*}}}*/
 950
 951 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 952 // ---------------------------------------------------------------------
 953 /* */
 954 ReleaseWriter::ReleaseWriter(string const &DB)
 955 {
 956    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 957    {
 958       AddPattern("Packages");
 959       AddPattern("Packages.gz");
 960       AddPattern("Packages.bz2");
 961       AddPattern("Packages.lzma");
 962       AddPattern("Packages.xz");
 963       AddPattern("Translation-*");
 964       AddPattern("Sources");
 965       AddPattern("Sources.gz");
 966       AddPattern("Sources.bz2");
 967       AddPattern("Sources.lzma");
 968       AddPattern("Sources.xz");
 969       AddPattern("Release");
 970       AddPattern("Contents-*");
 971       AddPattern("Index");
 972       AddPattern("md5sum.txt");
 973    }
 974    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 975
 976    Output = stdout;
 977    time_t const now = time(NULL);
 978
 979    setlocale(LC_TIME, "C");
 980
 981    char datestr[128];
 982    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 983                 gmtime(&now)) == 0)
 984    {
 985       datestr[0] = '\0';
 986    }
 987
 988    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 989    char validstr[128];
 990    if (now == validuntil ||
 991        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 992                 gmtime(&validuntil)) == 0)
 993    {
 994       validstr[0] = '\0';
 995    }
 996
 997    setlocale(LC_TIME, "");
 998
 999    map<string,string> Fields;
1000    Fields["Origin"] = "";
1001    Fields["Label"] = "";
1002    Fields["Suite"] = "";
1003    Fields["Version"] = "";
1004    Fields["Codename"] = "";
1005    Fields["Date"] = datestr;
1006    Fields["Valid-Until"] = validstr;
1007    Fields["Architectures"] = "";
1008    Fields["Components"] = "";
1009    Fields["Description"] = "";
1010
1011    for(map<string,string>::const_iterator I = Fields.begin();
1012        I != Fields.end();
1013        ++I)
1014    {
1015       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1016       string Value = _config->Find(Config, (*I).second.c_str());
1017       if (Value == "")
1018          continue;
1019
1020       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1021    }
1022
1023    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1024    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1025    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1026 }
1027                                                                         /*}}}*/
1028 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1029 // ---------------------------------------------------------------------
1030 bool ReleaseWriter::DoPackage(string FileName)
1031 {
1032    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1033    string NewFileName;
1034    if (DirStrip.empty() == false &&
1035        FileName.length() > DirStrip.length() &&
1036        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1037                  DirStrip.begin(),DirStrip.end()) == 0)
1038    {
1039       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1040       while (NewFileName[0] == '/')
1041          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1042    }
1043    else
1044       NewFileName = FileName;
1045
1046    if (PathPrefix.empty() == false)
1047       NewFileName = flCombine(PathPrefix,NewFileName);
1048
1049    FileFd fd(FileName, FileFd::ReadOnly);
1050
1051    if (!fd.IsOpen())
1052    {
1053       return false;
1054    }
1055
1056    CheckSums[NewFileName].size = fd.Size();
1057
1058    Hashes hs;
1059    hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1060    if (DoMD5 == true)
1061       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1062    if (DoSHA1 == true)
1063       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1064    if (DoSHA256 == true)
1065       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1066    if (DoSHA512 == true)
1067       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1068    fd.Close();
1069
1070    return true;
1071 }
1072
1073                                                                         /*}}}*/
1074 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1075 // ---------------------------------------------------------------------
1076 void ReleaseWriter::Finish()
1077 {
1078    if (DoMD5 == true)
1079    {
1080       fprintf(Output, "MD5Sum:\n");
1081       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1082           I != CheckSums.end(); ++I)
1083       {
1084          fprintf(Output, " %s %16llu %s\n",
1085                  (*I).second.MD5.c_str(),
1086                  (*I).second.size,
1087                  (*I).first.c_str());
1088       }
1089    }
1090    if (DoSHA1 == true)
1091    {
1092       fprintf(Output, "SHA1:\n");
1093       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1094           I != CheckSums.end(); ++I)
1095       {
1096          fprintf(Output, " %s %16llu %s\n",
1097                  (*I).second.SHA1.c_str(),
1098                  (*I).second.size,
1099                  (*I).first.c_str());
1100       }
1101    }
1102    if (DoSHA256 == true)
1103    {
1104       fprintf(Output, "SHA256:\n");
1105       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1106           I != CheckSums.end(); ++I)
1107       {
1108          fprintf(Output, " %s %16llu %s\n",
1109                  (*I).second.SHA256.c_str(),
1110                  (*I).second.size,
1111                  (*I).first.c_str());
1112       }
1113    }
1114
1115    fprintf(Output, "SHA512:\n");
1116    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1117        I != CheckSums.end();
1118        ++I)
1119    {
1120       fprintf(Output, " %s %16llu %s\n",
1121               (*I).second.SHA512.c_str(),
1122               (*I).second.size,
1123               (*I).first.c_str());
1124    }
1125
1126 }