ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/error.h>
  18 #include <apt-pkg/configuration.h>
  19 #include <apt-pkg/aptconfiguration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/deblistparser.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/gpgv.h>
  25
  26 #include <sys/types.h>
  27 #include <unistd.h>
  28 #include <ctime>
  29 #include <ftw.h>
  30 #include <fnmatch.h>
  31 #include <iostream>
  32 #include <sstream>
  33 #include <memory>
  34
  35 #include "writer.h"
  36 #include "cachedb.h"
  37 #include "apt-ftparchive.h"
  38 #include "multicompress.h"
  39
  40 #include <apti18n.h>
  41                                                                         /*}}}*/
  42 using namespace std;
  43 FTWScanner *FTWScanner::Owner;
  44
  45 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  46 // ---------------------------------------------------------------------
  47 /* */
  48 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  49                              const char *tag,
  50                              const char *rewrite,
  51                              const char *newtag = 0)
  52 {
  53     tfrd.Tag = tag;
  54     tfrd.Rewrite = rewrite;
  55     tfrd.NewTag = newtag;
  56 }
  57                                                                         /*}}}*/
  58
  59 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  63 {
  64    ErrorPrinted = false;
  65    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  66
  67    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  68    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  69    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  70    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  71 }
  72                                                                         /*}}}*/
  73 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  74 // ---------------------------------------------------------------------
  75 /* This is the FTW scanner, it processes each directory element in the
  76    directory tree. */
  77 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  78 {
  79    if (Flag == FTW_DNR)
  80    {
  81       Owner->NewLine(1);
  82       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  83    }
  84    if (Flag == FTW_NS)
  85    {
  86       Owner->NewLine(1);
  87       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  88    }
  89    if (Flag != FTW_F)
  90       return 0;
  91
  92    return ScannerFile(File, true);
  93 }
  94                                                                         /*}}}*/
  95 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* */
  98 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  99 {
 100    const char *LastComponent = strrchr(File, '/');
 101    char *RealPath = NULL;
 102
 103    if (LastComponent == NULL)
 104       LastComponent = File;
 105    else
 106       LastComponent++;
 107
 108    vector<string>::const_iterator I;
 109    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 110    {
 111       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 112          break;
 113    }
 114    if (I == Owner->Patterns.end())
 115       return 0;
 116
 117    /* Process it. If the file is a link then resolve it into an absolute
 118       name.. This works best if the directory components the scanner are
 119       given are not links themselves. */
 120    char Jnk[2];
 121    Owner->OriginalPath = File;
 122    if (ReadLink &&
 123        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 124        (RealPath = realpath(File,NULL)) != 0)
 125    {
 126       Owner->DoPackage(RealPath);
 127       free(RealPath);
 128    }
 129    else
 130       Owner->DoPackage(File);
 131
 132    if (_error->empty() == false)
 133    {
 134       // Print any errors or warnings found
 135       string Err;
 136       bool SeenPath = false;
 137       while (_error->empty() == false)
 138       {
 139          Owner->NewLine(1);
 140
 141          bool const Type = _error->PopMessage(Err);
 142          if (Type == true)
 143             cerr << _("E: ") << Err << endl;
 144          else
 145             cerr << _("W: ") << Err << endl;
 146
 147          if (Err.find(File) != string::npos)
 148             SeenPath = true;
 149       }
 150
 151       if (SeenPath == false)
 152          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 153       return 0;
 154    }
 155
 156    return 0;
 157 }
 158                                                                         /*}}}*/
 159 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 160 // ---------------------------------------------------------------------
 161 /* */
 162 bool FTWScanner::RecursiveScan(string const &Dir)
 163 {
 164    char *RealPath = NULL;
 165    /* If noprefix is set then jam the scan root in, so we don't generate
 166       link followed paths out of control */
 167    if (InternalPrefix.empty() == true)
 168    {
 169       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 170          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 171       InternalPrefix = RealPath;
 172       free(RealPath);
 173    }
 174
 175    // Do recursive directory searching
 176    Owner = this;
 177    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 178
 179    // Error treewalking?
 180    if (Res != 0)
 181    {
 182       if (_error->PendingError() == false)
 183          _error->Errno("ftw",_("Tree walking failed"));
 184       return false;
 185    }
 186
 187    return true;
 188 }
 189                                                                         /*}}}*/
 190 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 191 // ---------------------------------------------------------------------
 192 /* This is an alternative to using FTW to locate files, it reads the list
 193    of files from another file. */
 194 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 195 {
 196    char *RealPath = NULL;
 197    /* If noprefix is set then jam the scan root in, so we don't generate
 198       link followed paths out of control */
 199    if (InternalPrefix.empty() == true)
 200    {
 201       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 202          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 203       InternalPrefix = RealPath;
 204       free(RealPath);
 205    }
 206
 207    Owner = this;
 208    FILE *List = fopen(File.c_str(),"r");
 209    if (List == 0)
 210       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 211
 212    /* We are a tad tricky here.. We prefix the buffer with the directory
 213       name, that way if we need a full path with just use line.. Sneaky and
 214       fully evil. */
 215    char Line[1000];
 216    char *FileStart;
 217    if (Dir.empty() == true || Dir.end()[-1] != '/')
 218       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 219    else
 220       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 221    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 222    {
 223       char *FileName = _strstrip(FileStart);
 224       if (FileName[0] == 0)
 225          continue;
 226
 227       if (FileName[0] != '/')
 228       {
 229          if (FileName != FileStart)
 230             memmove(FileStart,FileName,strlen(FileStart));
 231          FileName = Line;
 232       }
 233
 234 #if 0
 235       struct stat St;
 236       int Flag = FTW_F;
 237       if (stat(FileName,&St) != 0)
 238          Flag = FTW_NS;
 239 #endif
 240
 241       if (ScannerFile(FileName, false) != 0)
 242          break;
 243    }
 244
 245    fclose(List);
 246    return true;
 247 }
 248                                                                         /*}}}*/
 249 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 250 // ---------------------------------------------------------------------
 251 /* */
 252 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 253                         unsigned long long &DeLinkBytes,
 254                         unsigned long long const &FileSize)
 255 {
 256    // See if this isn't an internaly prefix'd file name.
 257    if (InternalPrefix.empty() == false &&
 258        InternalPrefix.length() < FileName.length() &&
 259        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 260                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 261    {
 262       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 263       {
 264          // Tidy up the display
 265          if (DeLinkBytes == 0)
 266             cout << endl;
 267
 268          NewLine(1);
 269          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 270                     SizeToStr(FileSize).c_str());
 271          c1out << flush;
 272
 273          if (NoLinkAct == false)
 274          {
 275             char OldLink[400];
 276             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 277                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 278             else
 279             {
 280                if (unlink(OriginalPath) != 0)
 281                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 282                else
 283                {
 284                   if (link(FileName.c_str(),OriginalPath) != 0)
 285                   {
 286                      // Panic! Restore the symlink
 287                      symlink(OldLink,OriginalPath);
 288                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 289                                           FileName.c_str(),
 290                                           OriginalPath);
 291                   }
 292                }
 293             }
 294          }
 295
 296          DeLinkBytes += FileSize;
 297          if (DeLinkBytes/1024 >= DeLinkLimit)
 298             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 299       }
 300
 301       FileName = OriginalPath;
 302    }
 303
 304    return true;
 305 }
 306                                                                         /*}}}*/
 307
 308 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 309 // ---------------------------------------------------------------------
 310 /* */
 311 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 312                                string const &Arch) :
 313    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 314 {
 315    Output = stdout;
 316    SetExts(".deb .udeb");
 317    DeLinkLimit = 0;
 318
 319    // Process the command line options
 320    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 321    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 322    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 323    DoSHA512 = _config->FindB("APT::FTPArchive::Packages::SHA512",DoSHA512);
 324    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 325    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 326    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 327    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 328
 329    if (Db.Loaded() == false)
 330       DoContents = false;
 331
 332    // Read the override file
 333    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 334       return;
 335    else
 336       NoOverride = true;
 337
 338    if (ExtOverrides.empty() == false)
 339       Over.ReadExtraOverride(ExtOverrides);
 340
 341    _error->DumpErrors();
 342 }
 343                                                                         /*}}}*/
 344 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 345 // ---------------------------------------------------------------------
 346 /* */
 347 bool FTWScanner::SetExts(string const &Vals)
 348 {
 349    ClearPatterns();
 350    string::size_type Start = 0;
 351    while (Start <= Vals.length()-1)
 352    {
 353       string::size_type const Space = Vals.find(' ',Start);
 354       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 355       if ( Arch.empty() == false )
 356       {
 357          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 358          AddPattern(string("*_all") + Vals.substr(Start, Length));
 359       }
 360       else
 361          AddPattern(string("*") + Vals.substr(Start, Length));
 362
 363       Start += Length + 1;
 364    }
 365
 366    return true;
 367 }
 368
 369                                                                         /*}}}*/
 370 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 371 // ---------------------------------------------------------------------
 372 /* This method takes a package and gets its control information and
 373    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 374    rewritten and the path/size/hash appended. */
 375 bool PackagesWriter::DoPackage(string FileName)
 376 {
 377    // Pull all the data we need form the DB
 378    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 379                   == false)
 380    {
 381       return false;
 382    }
 383
 384    unsigned long long FileSize = Db.GetFileSize();
 385    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 386       return false;
 387
 388    // Lookup the overide information
 389    pkgTagSection &Tags = Db.Control.Section;
 390    string Package = Tags.FindS("Package");
 391    string Architecture;
 392    // if we generate a Packages file for a given arch, we use it to
 393    // look for overrides. if we run in "simple" mode without the
 394    // "Architecures" variable in the config we use the architecure value
 395    // from the deb file
 396    if(Arch != "")
 397       Architecture = Arch;
 398    else
 399       Architecture = Tags.FindS("Architecture");
 400    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 401
 402    if (Package.empty() == true)
 403       return _error->Error(_("Archive had no package field"));
 404
 405    // If we need to do any rewriting of the header do it now..
 406    if (OverItem.get() == 0)
 407    {
 408       if (NoOverride == false)
 409       {
 410          NewLine(1);
 411          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 412       }
 413
 414       OverItem = auto_ptr<Override::Item>(new Override::Item);
 415       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 416       OverItem->Priority = Tags.FindS("Priority");
 417    }
 418
 419    char Size[40];
 420    sprintf(Size,"%llu", (unsigned long long) FileSize);
 421
 422    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 423    string NewFileName;
 424    if (DirStrip.empty() == false &&
 425        FileName.length() > DirStrip.length() &&
 426        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 427                  DirStrip.begin(),DirStrip.end()) == 0)
 428       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 429    else
 430       NewFileName = FileName;
 431    if (PathPrefix.empty() == false)
 432       NewFileName = flCombine(PathPrefix,NewFileName);
 433
 434    /* Configuration says we don't want to include the long Description
 435       in the package file - instead we want to ship a separated file */
 436    string desc;
 437    if (LongDescription == false) {
 438       desc = Tags.FindS("Description").append("\n");
 439       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 440    }
 441
 442    // This lists all the changes to the fields we are going to make.
 443    // (7 hardcoded + maintainer + suggests + end marker)
 444    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 445
 446    unsigned int End = 0;
 447    SetTFRewriteData(Changes[End++], "Size", Size);
 448    if (DoMD5 == true)
 449       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 450    if (DoSHA1 == true)
 451       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 452    if (DoSHA256 == true)
 453       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 454    if (DoSHA512 == true)
 455       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 456    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 457    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 458    SetTFRewriteData(Changes[End++], "Status", 0);
 459    SetTFRewriteData(Changes[End++], "Optional", 0);
 460
 461    string DescriptionMd5;
 462    if (LongDescription == false) {
 463       MD5Summation descmd5;
 464       descmd5.Add(desc.c_str());
 465       DescriptionMd5 = descmd5.Result().Value();
 466       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 467       if (TransWriter != NULL)
 468          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 469    }
 470
 471    // Rewrite the maintainer field if necessary
 472    bool MaintFailed;
 473    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 474    if (MaintFailed == true)
 475    {
 476       if (NoOverride == false)
 477       {
 478          NewLine(1);
 479          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 480                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 481       }
 482    }
 483
 484    if (NewMaint.empty() == false)
 485       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 486
 487    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 488       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 489       but dpkg does this append bit. So we do the append bit, at least that way the
 490       status file and package file will remain similar. There are other transforms
 491       but optional is the only legacy one still in use for some lazy reason. */
 492    string OptionalStr = Tags.FindS("Optional");
 493    if (OptionalStr.empty() == false)
 494    {
 495       if (Tags.FindS("Suggests").empty() == false)
 496          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 497       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 498    }
 499
 500    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 501         I != OverItem->FieldOverride.end(); ++I)
 502       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 503
 504    SetTFRewriteData(Changes[End++], 0, 0);
 505
 506    // Rewrite and store the fields.
 507    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 508       return false;
 509    fprintf(Output,"\n");
 510
 511    return Db.Finish();
 512 }
 513                                                                         /*}}}*/
 514
 515 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 516 // ---------------------------------------------------------------------
 517 /* Create a Translation-Master file for this Packages file */
 518 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 519                                         mode_t const &Permissions) : Output(NULL),
 520                                                         RefCounter(0)
 521 {
 522    if (File.empty() == true)
 523       return;
 524
 525    Comp = new MultiCompress(File, TransCompress, Permissions);
 526    Output = Comp->Input;
 527 }
 528                                                                         /*}}}*/
 529 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* Create a Translation-Master file for this Packages file */
 532 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 533                                   string const &MD5)
 534 {
 535    if (Output == NULL)
 536       return true;
 537
 538    // Different archs can include different versions and therefore
 539    // different descriptions - so we need to check for both name and md5.
 540    string const Record = Pkg + ":" + MD5;
 541
 542    if (Included.find(Record) != Included.end())
 543       return true;
 544
 545    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 546            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 547
 548    Included.insert(Record);
 549    return true;
 550 }
 551                                                                         /*}}}*/
 552 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* */
 555 TranslationWriter::~TranslationWriter()
 556 {
 557    if (Comp == NULL)
 558       return;
 559
 560    delete Comp;
 561 }
 562                                                                         /*}}}*/
 563
 564 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 565 // ---------------------------------------------------------------------
 566 /* */
 567 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 568                              string const &ExtOverrides)
 569 {
 570    Output = stdout;
 571    AddPattern("*.dsc");
 572    DeLinkLimit = 0;
 573    Buffer = 0;
 574    BufSize = 0;
 575
 576    // Process the command line options
 577    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 578    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 579    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 580    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 581
 582    // Read the override file
 583    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 584       return;
 585    else
 586       NoOverride = true;
 587
 588    // WTF?? The logic above: if we can't read binary overrides, don't even try
 589    // reading source overrides. if we can read binary overrides, then say there
 590    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 591
 592    if (ExtOverrides.empty() == false)
 593       SOver.ReadExtraOverride(ExtOverrides);
 594
 595    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 596       SOver.ReadOverride(SOverrides,true);
 597 }
 598                                                                         /*}}}*/
 599 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* */
 602 bool SourcesWriter::DoPackage(string FileName)
 603 {
 604    // Open the archive
 605    FileFd F;
 606    if (OpenMaybeClearSignedFile(FileName, F) == false)
 607       return false;
 608
 609    unsigned long long const FSize = F.FileSize();
 610    //FIXME: do we really need to enforce a maximum size of the dsc file?
 611    if (FSize > 128*1024)
 612       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 613
 614    if (BufSize < FSize + 2)
 615    {
 616       BufSize = FSize + 2;
 617       Buffer = (char *)realloc(Buffer , BufSize);
 618    }
 619
 620    if (F.Read(Buffer, FSize) == false)
 621       return false;
 622
 623    // Stat the file for later (F might be clearsigned, so not F.FileSize())
 624    struct stat St;
 625    if (stat(FileName.c_str(), &St) != 0)
 626       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 627
 628    // Hash the file
 629    char *Start = Buffer;
 630    char *BlkEnd = Buffer + FSize;
 631
 632    Hashes DscHashes;
 633    if (FSize == (unsigned long long) St.st_size)
 634    {
 635       if (DoMD5 == true)
 636          DscHashes.MD5.Add((unsigned char *)Start,BlkEnd - Start);
 637       if (DoSHA1 == true)
 638          DscHashes.SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 639       if (DoSHA256 == true)
 640          DscHashes.SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 641       if (DoSHA512 == true)
 642          DscHashes.SHA512.Add((unsigned char *)Start,BlkEnd - Start);
 643    }
 644    else
 645    {
 646       FileFd DscFile(FileName, FileFd::ReadOnly);
 647       DscHashes.AddFD(DscFile, St.st_size, DoMD5, DoSHA1, DoSHA256, DoSHA512);
 648    }
 649
 650    // Add extra \n to the end, just in case (as in clearsigned they are missing)
 651    *BlkEnd++ = '\n';
 652    *BlkEnd++ = '\n';
 653
 654    pkgTagSection Tags;
 655    if (Tags.Scan(Start,BlkEnd - Start) == false || Tags.Exists("Source") == false)
 656       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 657    Tags.Trim();
 658
 659    // Lookup the overide information, finding first the best priority.
 660    string BestPrio;
 661    string Bins = Tags.FindS("Binary");
 662    char Buffer[Bins.length() + 1];
 663    auto_ptr<Override::Item> OverItem(0);
 664    if (Bins.empty() == false)
 665    {
 666       strcpy(Buffer,Bins.c_str());
 667
 668       // Ignore too-long errors.
 669       char *BinList[400];
 670       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 671
 672       // Look at all the binaries
 673       unsigned char BestPrioV = pkgCache::State::Extra;
 674       for (unsigned I = 0; BinList[I] != 0; I++)
 675       {
 676          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 677          if (Itm.get() == 0)
 678             continue;
 679
 680          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 681          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 682          {
 683             BestPrioV = NewPrioV;
 684             BestPrio = Itm->Priority;
 685          }
 686
 687          if (OverItem.get() == 0)
 688             OverItem = Itm;
 689       }
 690    }
 691
 692    // If we need to do any rewriting of the header do it now..
 693    if (OverItem.get() == 0)
 694    {
 695       if (NoOverride == false)
 696       {
 697          NewLine(1);
 698          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 699       }
 700
 701       OverItem = auto_ptr<Override::Item>(new Override::Item);
 702    }
 703
 704    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 705    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 706    if (SOverItem.get() == 0)
 707    {
 708       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 709       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 710       if (SOverItem.get() == 0)
 711       {
 712         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 713          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 714          *SOverItem = *OverItem;
 715       }
 716    }
 717
 718    // Add the dsc to the files hash list
 719    string const strippedName = flNotDir(FileName);
 720    std::ostringstream ostreamFiles;
 721    if (DoMD5 == true && Tags.Exists("Files"))
 722       ostreamFiles << "\n " << string(DscHashes.MD5.Result()) << " " << St.st_size << " "
 723                    << strippedName << "\n " << Tags.FindS("Files");
 724    string const Files = ostreamFiles.str();
 725
 726    std::ostringstream ostreamSha1;
 727    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 728       ostreamSha1 << "\n " << string(DscHashes.SHA1.Result()) << " " << St.st_size << " "
 729                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 730    string const ChecksumsSha1 = ostreamSha1.str();
 731
 732    std::ostringstream ostreamSha256;
 733    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 734       ostreamSha256 << "\n " << string(DscHashes.SHA256.Result()) << " " << St.st_size << " "
 735                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 736    string const ChecksumsSha256 = ostreamSha256.str();
 737
 738    std::ostringstream ostreamSha512;
 739    if (Tags.Exists("Checksums-Sha512"))
 740       ostreamSha512 << "\n " << string(DscHashes.SHA512.Result()) << " " << St.st_size << " "
 741                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 742    string const ChecksumsSha512 = ostreamSha512.str();
 743
 744    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 745    string NewFileName;
 746    if (DirStrip.empty() == false &&
 747        FileName.length() > DirStrip.length() &&
 748        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 749       NewFileName = string(OriginalPath + DirStrip.length());
 750    else
 751       NewFileName = OriginalPath;
 752    if (PathPrefix.empty() == false)
 753       NewFileName = flCombine(PathPrefix,NewFileName);
 754
 755    string Directory = flNotFile(OriginalPath);
 756    string Package = Tags.FindS("Source");
 757
 758    // Perform the delinking operation over all of the files
 759    string ParseJnk;
 760    const char *C = Files.c_str();
 761    char *RealPath = NULL;
 762    for (;isspace(*C); C++);
 763    while (*C != 0)
 764    {
 765       // Parse each of the elements
 766       if (ParseQuoteWord(C,ParseJnk) == false ||
 767           ParseQuoteWord(C,ParseJnk) == false ||
 768           ParseQuoteWord(C,ParseJnk) == false)
 769          return _error->Error("Error parsing file record");
 770
 771       char Jnk[2];
 772       string OriginalPath = Directory + ParseJnk;
 773       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 774           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 775       {
 776          string RP = RealPath;
 777          free(RealPath);
 778          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 779             return false;
 780       }
 781    }
 782
 783    Directory = flNotFile(NewFileName);
 784    if (Directory.length() > 2)
 785       Directory.erase(Directory.end()-1);
 786
 787    // This lists all the changes to the fields we are going to make.
 788    // (5 hardcoded + checksums + maintainer + end marker)
 789    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 790
 791    unsigned int End = 0;
 792    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 793    if (Files.empty() == false)
 794       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 795    if (ChecksumsSha1.empty() == false)
 796       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 797    if (ChecksumsSha256.empty() == false)
 798       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 799    if (ChecksumsSha512.empty() == false)
 800       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 801    if (Directory != "./")
 802       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 803    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 804    SetTFRewriteData(Changes[End++],"Status",0);
 805
 806    // Rewrite the maintainer field if necessary
 807    bool MaintFailed;
 808    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 809    if (MaintFailed == true)
 810    {
 811       if (NoOverride == false)
 812       {
 813          NewLine(1);
 814          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 815                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 816       }
 817    }
 818    if (NewMaint.empty() == false)
 819       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 820
 821    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 822         I != SOverItem->FieldOverride.end(); ++I)
 823       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 824
 825    SetTFRewriteData(Changes[End++], 0, 0);
 826
 827    // Rewrite and store the fields.
 828    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 829       return false;
 830    fprintf(Output,"\n");
 831
 832    Stats.Packages++;
 833
 834    return true;
 835 }
 836                                                                         /*}}}*/
 837
 838 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 839 // ---------------------------------------------------------------------
 840 /* */
 841 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 842                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 843
 844 {
 845    SetExts(".deb");
 846    Output = stdout;
 847 }
 848                                                                         /*}}}*/
 849 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 850 // ---------------------------------------------------------------------
 851 /* If Package is the empty string the control record will be parsed to
 852    determine what the package name is. */
 853 bool ContentsWriter::DoPackage(string FileName, string Package)
 854 {
 855    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 856    {
 857       return false;
 858    }
 859
 860    // Parse the package name
 861    if (Package.empty() == true)
 862    {
 863       Package = Db.Control.Section.FindS("Package");
 864    }
 865
 866    Db.Contents.Add(Gen,Package);
 867
 868    return Db.Finish();
 869 }
 870                                                                         /*}}}*/
 871 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 872 // ---------------------------------------------------------------------
 873 /* */
 874 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 875 {
 876    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 877    if (_error->PendingError() == true)
 878       return false;
 879
 880    // Open the package file
 881    FileFd Fd;
 882    if (Pkgs.OpenOld(Fd) == false)
 883       return false;
 884
 885    pkgTagFile Tags(&Fd);
 886    if (_error->PendingError() == true)
 887       return false;
 888
 889    // Parse.
 890    pkgTagSection Section;
 891    while (Tags.Step(Section) == true)
 892    {
 893       string File = flCombine(Prefix,Section.FindS("FileName"));
 894       string Package = Section.FindS("Section");
 895       if (Package.empty() == false && Package.end()[-1] != '/')
 896       {
 897          Package += '/';
 898          Package += Section.FindS("Package");
 899       }
 900       else
 901          Package += Section.FindS("Package");
 902
 903       DoPackage(File,Package);
 904       if (_error->empty() == false)
 905       {
 906          _error->Error("Errors apply to file '%s'",File.c_str());
 907          _error->DumpErrors();
 908       }
 909    }
 910
 911    // Tidy the compressor
 912    Fd.Close();
 913
 914    return true;
 915 }
 916
 917                                                                         /*}}}*/
 918
 919 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 920 // ---------------------------------------------------------------------
 921 /* */
 922 ReleaseWriter::ReleaseWriter(string const &DB)
 923 {
 924    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 925    {
 926       AddPattern("Packages");
 927       AddPattern("Packages.gz");
 928       AddPattern("Packages.bz2");
 929       AddPattern("Packages.lzma");
 930       AddPattern("Packages.xz");
 931       AddPattern("Translation-*");
 932       AddPattern("Sources");
 933       AddPattern("Sources.gz");
 934       AddPattern("Sources.bz2");
 935       AddPattern("Sources.lzma");
 936       AddPattern("Sources.xz");
 937       AddPattern("Release");
 938       AddPattern("Contents-*");
 939       AddPattern("Index");
 940       AddPattern("md5sum.txt");
 941    }
 942    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 943
 944    Output = stdout;
 945    time_t const now = time(NULL);
 946
 947    setlocale(LC_TIME, "C");
 948
 949    char datestr[128];
 950    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 951                 gmtime(&now)) == 0)
 952    {
 953       datestr[0] = '\0';
 954    }
 955
 956    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 957    char validstr[128];
 958    if (now == validuntil ||
 959        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 960                 gmtime(&validuntil)) == 0)
 961    {
 962       validstr[0] = '\0';
 963    }
 964
 965    setlocale(LC_TIME, "");
 966
 967    map<string,string> Fields;
 968    Fields["Origin"] = "";
 969    Fields["Label"] = "";
 970    Fields["Suite"] = "";
 971    Fields["Version"] = "";
 972    Fields["Codename"] = "";
 973    Fields["Date"] = datestr;
 974    Fields["Valid-Until"] = validstr;
 975    Fields["Architectures"] = "";
 976    Fields["Components"] = "";
 977    Fields["Description"] = "";
 978
 979    for(map<string,string>::const_iterator I = Fields.begin();
 980        I != Fields.end();
 981        ++I)
 982    {
 983       string Config = string("APT::FTPArchive::Release::") + (*I).first;
 984       string Value = _config->Find(Config, (*I).second.c_str());
 985       if (Value == "")
 986          continue;
 987
 988       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
 989    }
 990
 991    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
 992    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
 993    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
 994 }
 995                                                                         /*}}}*/
 996 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
 997 // ---------------------------------------------------------------------
 998 bool ReleaseWriter::DoPackage(string FileName)
 999 {
1000    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1001    string NewFileName;
1002    if (DirStrip.empty() == false &&
1003        FileName.length() > DirStrip.length() &&
1004        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1005                  DirStrip.begin(),DirStrip.end()) == 0)
1006    {
1007       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1008       while (NewFileName[0] == '/')
1009          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1010    }
1011    else
1012       NewFileName = FileName;
1013
1014    if (PathPrefix.empty() == false)
1015       NewFileName = flCombine(PathPrefix,NewFileName);
1016
1017    FileFd fd(FileName, FileFd::ReadOnly);
1018
1019    if (!fd.IsOpen())
1020    {
1021       return false;
1022    }
1023
1024    CheckSums[NewFileName].size = fd.Size();
1025
1026    Hashes hs;
1027    hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1028    if (DoMD5 == true)
1029       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1030    if (DoSHA1 == true)
1031       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1032    if (DoSHA256 == true)
1033       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1034    if (DoSHA512 == true)
1035       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1036    fd.Close();
1037
1038    return true;
1039 }
1040
1041                                                                         /*}}}*/
1042 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1043 // ---------------------------------------------------------------------
1044 void ReleaseWriter::Finish()
1045 {
1046    if (DoMD5 == true)
1047    {
1048       fprintf(Output, "MD5Sum:\n");
1049       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1050           I != CheckSums.end(); ++I)
1051       {
1052          fprintf(Output, " %s %16llu %s\n",
1053                  (*I).second.MD5.c_str(),
1054                  (*I).second.size,
1055                  (*I).first.c_str());
1056       }
1057    }
1058    if (DoSHA1 == true)
1059    {
1060       fprintf(Output, "SHA1:\n");
1061       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1062           I != CheckSums.end(); ++I)
1063       {
1064          fprintf(Output, " %s %16llu %s\n",
1065                  (*I).second.SHA1.c_str(),
1066                  (*I).second.size,
1067                  (*I).first.c_str());
1068       }
1069    }
1070    if (DoSHA256 == true)
1071    {
1072       fprintf(Output, "SHA256:\n");
1073       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1074           I != CheckSums.end(); ++I)
1075       {
1076          fprintf(Output, " %s %16llu %s\n",
1077                  (*I).second.SHA256.c_str(),
1078                  (*I).second.size,
1079                  (*I).first.c_str());
1080       }
1081    }
1082
1083    fprintf(Output, "SHA512:\n");
1084    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1085        I != CheckSums.end();
1086        ++I)
1087    {
1088       fprintf(Output, " %s %16llu %s\n",
1089               (*I).second.SHA512.c_str(),
1090               (*I).second.size,
1091               (*I).first.c_str());
1092    }
1093
1094 }