ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include "writer.h"
  15
  16 #include <apti18n.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/sha1.h>
  22 #include <apt-pkg/sha256.h>
  23 #include <apt-pkg/deblistparser.h>
  24
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <ctime>
  28 #include <ftw.h>
  29 #include <fnmatch.h>
  30 #include <iostream>
  31 #include <sstream>
  32 #include <memory>
  33
  34 #include "cachedb.h"
  35 #include "apt-ftparchive.h"
  36 #include "multicompress.h"
  37                                                                         /*}}}*/
  38 using namespace std;
  39 FTWScanner *FTWScanner::Owner;
  40
  41 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  45                              const char *tag,
  46                              const char *rewrite,
  47                              const char *newtag = 0)
  48 {
  49     tfrd.Tag = tag;
  50     tfrd.Rewrite = rewrite;
  51     tfrd.NewTag = newtag;
  52 }
  53                                                                         /*}}}*/
  54
  55 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  59 {
  60    ErrorPrinted = false;
  61    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  62
  63    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  64    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  65    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  66 }
  67                                                                         /*}}}*/
  68 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* This is the FTW scanner, it processes each directory element in the
  71    directory tree. */
  72 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  73 {
  74    if (Flag == FTW_DNR)
  75    {
  76       Owner->NewLine(1);
  77       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  78    }
  79    if (Flag == FTW_NS)
  80    {
  81       Owner->NewLine(1);
  82       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  83    }
  84    if (Flag != FTW_F)
  85       return 0;
  86
  87    return ScannerFile(File, true);
  88 }
  89                                                                         /*}}}*/
  90 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  91 // ---------------------------------------------------------------------
  92 /* */
  93 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  94 {
  95    const char *LastComponent = strrchr(File, '/');
  96    char *RealPath = NULL;
  97
  98    if (LastComponent == NULL)
  99       LastComponent = File;
 100    else
 101       LastComponent++;
 102
 103    vector<string>::const_iterator I;
 104    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 105    {
 106       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 107          break;
 108    }
 109    if (I == Owner->Patterns.end())
 110       return 0;
 111
 112    /* Process it. If the file is a link then resolve it into an absolute
 113       name.. This works best if the directory components the scanner are
 114       given are not links themselves. */
 115    char Jnk[2];
 116    Owner->OriginalPath = File;
 117    if (ReadLink &&
 118        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 119        (RealPath = realpath(File,NULL)) != 0)
 120    {
 121       Owner->DoPackage(RealPath);
 122       free(RealPath);
 123    }
 124    else
 125       Owner->DoPackage(File);
 126
 127    if (_error->empty() == false)
 128    {
 129       // Print any errors or warnings found
 130       string Err;
 131       bool SeenPath = false;
 132       while (_error->empty() == false)
 133       {
 134          Owner->NewLine(1);
 135
 136          bool const Type = _error->PopMessage(Err);
 137          if (Type == true)
 138             cerr << _("E: ") << Err << endl;
 139          else
 140             cerr << _("W: ") << Err << endl;
 141
 142          if (Err.find(File) != string::npos)
 143             SeenPath = true;
 144       }
 145
 146       if (SeenPath == false)
 147          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 148       return 0;
 149    }
 150
 151    return 0;
 152 }
 153                                                                         /*}}}*/
 154 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 155 // ---------------------------------------------------------------------
 156 /* */
 157 bool FTWScanner::RecursiveScan(string const &Dir)
 158 {
 159    char *RealPath = NULL;
 160    /* If noprefix is set then jam the scan root in, so we don't generate
 161       link followed paths out of control */
 162    if (InternalPrefix.empty() == true)
 163    {
 164       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 165          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 166       InternalPrefix = RealPath;
 167       free(RealPath);
 168    }
 169
 170    // Do recursive directory searching
 171    Owner = this;
 172    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 173
 174    // Error treewalking?
 175    if (Res != 0)
 176    {
 177       if (_error->PendingError() == false)
 178          _error->Errno("ftw",_("Tree walking failed"));
 179       return false;
 180    }
 181
 182    return true;
 183 }
 184                                                                         /*}}}*/
 185 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 186 // ---------------------------------------------------------------------
 187 /* This is an alternative to using FTW to locate files, it reads the list
 188    of files from another file. */
 189 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 190 {
 191    char *RealPath = NULL;
 192    /* If noprefix is set then jam the scan root in, so we don't generate
 193       link followed paths out of control */
 194    if (InternalPrefix.empty() == true)
 195    {
 196       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 197          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 198       InternalPrefix = RealPath;
 199       free(RealPath);
 200    }
 201
 202    Owner = this;
 203    FILE *List = fopen(File.c_str(),"r");
 204    if (List == 0)
 205       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 206
 207    /* We are a tad tricky here.. We prefix the buffer with the directory
 208       name, that way if we need a full path with just use line.. Sneaky and
 209       fully evil. */
 210    char Line[1000];
 211    char *FileStart;
 212    if (Dir.empty() == true || Dir.end()[-1] != '/')
 213       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 214    else
 215       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 216    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 217    {
 218       char *FileName = _strstrip(FileStart);
 219       if (FileName[0] == 0)
 220          continue;
 221
 222       if (FileName[0] != '/')
 223       {
 224          if (FileName != FileStart)
 225             memmove(FileStart,FileName,strlen(FileStart));
 226          FileName = Line;
 227       }
 228
 229 #if 0
 230       struct stat St;
 231       int Flag = FTW_F;
 232       if (stat(FileName,&St) != 0)
 233          Flag = FTW_NS;
 234 #endif
 235
 236       if (ScannerFile(FileName, false) != 0)
 237          break;
 238    }
 239
 240    fclose(List);
 241    return true;
 242 }
 243                                                                         /*}}}*/
 244 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 245 // ---------------------------------------------------------------------
 246 /* */
 247 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 248                         unsigned long &DeLinkBytes,
 249                         off_t const &FileSize)
 250 {
 251    // See if this isn't an internaly prefix'd file name.
 252    if (InternalPrefix.empty() == false &&
 253        InternalPrefix.length() < FileName.length() &&
 254        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 255                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 256    {
 257       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 258       {
 259          // Tidy up the display
 260          if (DeLinkBytes == 0)
 261             cout << endl;
 262
 263          NewLine(1);
 264          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 265                     SizeToStr(FileSize).c_str());
 266          c1out << flush;
 267
 268          if (NoLinkAct == false)
 269          {
 270             char OldLink[400];
 271             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 272                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 273             else
 274             {
 275                if (unlink(OriginalPath) != 0)
 276                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 277                else
 278                {
 279                   if (link(FileName.c_str(),OriginalPath) != 0)
 280                   {
 281                      // Panic! Restore the symlink
 282                      symlink(OldLink,OriginalPath);
 283                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 284                                           FileName.c_str(),
 285                                           OriginalPath);
 286                   }
 287                }
 288             }
 289          }
 290
 291          DeLinkBytes += FileSize;
 292          if (DeLinkBytes/1024 >= DeLinkLimit)
 293             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 294       }
 295
 296       FileName = OriginalPath;
 297    }
 298
 299    return true;
 300 }
 301                                                                         /*}}}*/
 302
 303 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 304 // ---------------------------------------------------------------------
 305 /* */
 306 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 307                                string const &Arch) :
 308    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 309 {
 310    Output = stdout;
 311    SetExts(".deb .udeb");
 312    DeLinkLimit = 0;
 313
 314    // Process the command line options
 315    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 316    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 317    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 318    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 319    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 320    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 321    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 322
 323    if (Db.Loaded() == false)
 324       DoContents = false;
 325
 326    // Read the override file
 327    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 328       return;
 329    else
 330       NoOverride = true;
 331
 332    if (ExtOverrides.empty() == false)
 333       Over.ReadExtraOverride(ExtOverrides);
 334
 335    _error->DumpErrors();
 336 }
 337                                                                         /*}}}*/
 338 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 339 // ---------------------------------------------------------------------
 340 /* */
 341 bool FTWScanner::SetExts(string const &Vals)
 342 {
 343    ClearPatterns();
 344    string::size_type Start = 0;
 345    while (Start <= Vals.length()-1)
 346    {
 347       string::size_type const Space = Vals.find(' ',Start);
 348       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 349       if ( Arch.empty() == false )
 350       {
 351          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 352          AddPattern(string("*_all") + Vals.substr(Start, Length));
 353       }
 354       else
 355          AddPattern(string("*") + Vals.substr(Start, Length));
 356
 357       Start += Length + 1;
 358    }
 359
 360    return true;
 361 }
 362
 363                                                                         /*}}}*/
 364 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* This method takes a package and gets its control information and
 367    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 368    rewritten and the path/size/hash appended. */
 369 bool PackagesWriter::DoPackage(string FileName)
 370 {
 371    // Pull all the data we need form the DB
 372    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoAlwaysStat)
 373                   == false)
 374    {
 375       return false;
 376    }
 377
 378    off_t FileSize = Db.GetFileSize();
 379    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 380       return false;
 381
 382    // Lookup the overide information
 383    pkgTagSection &Tags = Db.Control.Section;
 384    string Package = Tags.FindS("Package");
 385    string Architecture;
 386    // if we generate a Packages file for a given arch, we use it to
 387    // look for overrides. if we run in "simple" mode without the
 388    // "Architecures" variable in the config we use the architecure value
 389    // from the deb file
 390    if(Arch != "")
 391       Architecture = Arch;
 392    else
 393       Architecture = Tags.FindS("Architecture");
 394    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 395
 396    if (Package.empty() == true)
 397       return _error->Error(_("Archive had no package field"));
 398
 399    // If we need to do any rewriting of the header do it now..
 400    if (OverItem.get() == 0)
 401    {
 402       if (NoOverride == false)
 403       {
 404          NewLine(1);
 405          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 406       }
 407
 408       OverItem = auto_ptr<Override::Item>(new Override::Item);
 409       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 410       OverItem->Priority = Tags.FindS("Priority");
 411    }
 412
 413    char Size[40];
 414    sprintf(Size,"%lu", (unsigned long) FileSize);
 415
 416    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 417    string NewFileName;
 418    if (DirStrip.empty() == false &&
 419        FileName.length() > DirStrip.length() &&
 420        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 421                  DirStrip.begin(),DirStrip.end()) == 0)
 422       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 423    else
 424       NewFileName = FileName;
 425    if (PathPrefix.empty() == false)
 426       NewFileName = flCombine(PathPrefix,NewFileName);
 427
 428    /* Configuration says we don't want to include the long Description
 429       in the package file - instead we want to ship a separated file */
 430    string desc;
 431    if (LongDescription == false) {
 432       desc = Tags.FindS("Description").append("\n");
 433       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 434    }
 435
 436    // This lists all the changes to the fields we are going to make.
 437    // (7 hardcoded + maintainer + suggests + end marker)
 438    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 439
 440    unsigned int End = 0;
 441    SetTFRewriteData(Changes[End++], "Size", Size);
 442    if (DoMD5 == true)
 443       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 444    if (DoSHA1 == true)
 445       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 446    if (DoSHA256 == true)
 447       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 448    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 449    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 450    SetTFRewriteData(Changes[End++], "Status", 0);
 451    SetTFRewriteData(Changes[End++], "Optional", 0);
 452
 453    string DescriptionMd5;
 454    if (LongDescription == false) {
 455       MD5Summation descmd5;
 456       descmd5.Add(desc.c_str());
 457       DescriptionMd5 = descmd5.Result().Value();
 458       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 459       if (TransWriter != NULL)
 460          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 461    }
 462
 463    // Rewrite the maintainer field if necessary
 464    bool MaintFailed;
 465    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 466    if (MaintFailed == true)
 467    {
 468       if (NoOverride == false)
 469       {
 470          NewLine(1);
 471          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 472                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 473       }
 474    }
 475
 476    if (NewMaint.empty() == false)
 477       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 478
 479    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 480       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 481       but dpkg does this append bit. So we do the append bit, at least that way the
 482       status file and package file will remain similar. There are other transforms
 483       but optional is the only legacy one still in use for some lazy reason. */
 484    string OptionalStr = Tags.FindS("Optional");
 485    if (OptionalStr.empty() == false)
 486    {
 487       if (Tags.FindS("Suggests").empty() == false)
 488          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 489       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 490    }
 491
 492    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 493         I != OverItem->FieldOverride.end(); I++)
 494       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 495
 496    SetTFRewriteData(Changes[End++], 0, 0);
 497
 498    // Rewrite and store the fields.
 499    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 500       return false;
 501    fprintf(Output,"\n");
 502
 503    return Db.Finish();
 504 }
 505                                                                         /*}}}*/
 506
 507 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 508 // ---------------------------------------------------------------------
 509 /* Create a Translation-Master file for this Packages file */
 510 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 511                                         mode_t const &Permissions) : Output(NULL),
 512                                                         RefCounter(0)
 513 {
 514    if (File.empty() == true)
 515       return;
 516
 517    Comp = new MultiCompress(File, TransCompress, Permissions);
 518    Output = Comp->Input;
 519 }
 520                                                                         /*}}}*/
 521 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 522 // ---------------------------------------------------------------------
 523 /* Create a Translation-Master file for this Packages file */
 524 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 525                                   string const &MD5)
 526 {
 527    if (Output == NULL)
 528       return true;
 529
 530    // Different archs can include different versions and therefore
 531    // different descriptions - so we need to check for both name and md5.
 532    string const Record = Pkg + ":" + MD5;
 533
 534    if (Included.find(Record) != Included.end())
 535       return true;
 536
 537    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 538            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 539
 540    Included.insert(Record);
 541    return true;
 542 }
 543                                                                         /*}}}*/
 544 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 545 // ---------------------------------------------------------------------
 546 /* */
 547 TranslationWriter::~TranslationWriter()
 548 {
 549    if (Comp == NULL)
 550       return;
 551
 552    delete Comp;
 553 }
 554                                                                         /*}}}*/
 555
 556 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 557 // ---------------------------------------------------------------------
 558 /* */
 559 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 560                              string const &ExtOverrides)
 561 {
 562    Output = stdout;
 563    AddPattern("*.dsc");
 564    DeLinkLimit = 0;
 565    Buffer = 0;
 566    BufSize = 0;
 567
 568    // Process the command line options
 569    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 570    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 571    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 572    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 573
 574    // Read the override file
 575    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 576       return;
 577    else
 578       NoOverride = true;
 579
 580    // WTF?? The logic above: if we can't read binary overrides, don't even try
 581    // reading source overrides. if we can read binary overrides, then say there
 582    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 583
 584    if (ExtOverrides.empty() == false)
 585       SOver.ReadExtraOverride(ExtOverrides);
 586
 587    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 588       SOver.ReadOverride(SOverrides,true);
 589 }
 590                                                                         /*}}}*/
 591 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* */
 594 bool SourcesWriter::DoPackage(string FileName)
 595 {
 596    // Open the archive
 597    FileFd F(FileName,FileFd::ReadOnly);
 598    if (_error->PendingError() == true)
 599       return false;
 600
 601    // Stat the file for later
 602    struct stat St;
 603    if (fstat(F.Fd(),&St) != 0)
 604       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 605
 606    if (St.st_size > 128*1024)
 607       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 608
 609    if (BufSize < (unsigned)St.st_size+1)
 610    {
 611       BufSize = St.st_size+1;
 612       Buffer = (char *)realloc(Buffer,St.st_size+1);
 613    }
 614
 615    if (F.Read(Buffer,St.st_size) == false)
 616       return false;
 617
 618    // Hash the file
 619    char *Start = Buffer;
 620    char *BlkEnd = Buffer + St.st_size;
 621
 622    MD5Summation MD5;
 623    SHA1Summation SHA1;
 624    SHA256Summation SHA256;
 625
 626    if (DoMD5 == true)
 627       MD5.Add((unsigned char *)Start,BlkEnd - Start);
 628    if (DoSHA1 == true)
 629       SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 630    if (DoSHA256 == true)
 631       SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 632
 633    // Add an extra \n to the end, just in case
 634    *BlkEnd++ = '\n';
 635
 636    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 637       before */
 638    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 639    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 640    {
 641       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 642       {
 643          MsgEnd[1] = '\n';
 644          break;
 645       }
 646    }
 647
 648    /* Read records until we locate the Source record. This neatly skips the
 649       GPG header (which is RFC822 formed) without any trouble. */
 650    pkgTagSection Tags;
 651    do
 652    {
 653       unsigned Pos;
 654       if (Tags.Scan(Start,BlkEnd - Start) == false)
 655          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 656       if (Tags.Find("Source",Pos) == true)
 657          break;
 658       Start += Tags.size();
 659    }
 660    while (1);
 661    Tags.Trim();
 662
 663    // Lookup the overide information, finding first the best priority.
 664    string BestPrio;
 665    string Bins = Tags.FindS("Binary");
 666    char Buffer[Bins.length() + 1];
 667    auto_ptr<Override::Item> OverItem(0);
 668    if (Bins.empty() == false)
 669    {
 670       strcpy(Buffer,Bins.c_str());
 671
 672       // Ignore too-long errors.
 673       char *BinList[400];
 674       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 675
 676       // Look at all the binaries
 677       unsigned char BestPrioV = pkgCache::State::Extra;
 678       for (unsigned I = 0; BinList[I] != 0; I++)
 679       {
 680          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 681          if (Itm.get() == 0)
 682             continue;
 683
 684          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 685          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 686          {
 687             BestPrioV = NewPrioV;
 688             BestPrio = Itm->Priority;
 689          }
 690
 691          if (OverItem.get() == 0)
 692             OverItem = Itm;
 693       }
 694    }
 695
 696    // If we need to do any rewriting of the header do it now..
 697    if (OverItem.get() == 0)
 698    {
 699       if (NoOverride == false)
 700       {
 701          NewLine(1);
 702          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 703       }
 704
 705       OverItem = auto_ptr<Override::Item>(new Override::Item);
 706    }
 707
 708    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 709    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 710    if (SOverItem.get() == 0)
 711    {
 712       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 713       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 714       if (SOverItem.get() == 0)
 715       {
 716         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 717          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 718          *SOverItem = *OverItem;
 719       }
 720    }
 721
 722    // Add the dsc to the files hash list
 723    string const strippedName = flNotDir(FileName);
 724    std::ostringstream ostreamFiles;
 725    if (DoMD5 == true && Tags.Exists("Files"))
 726       ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 727                    << strippedName << "\n " << Tags.FindS("Files");
 728    string const Files = ostreamFiles.str();
 729
 730    std::ostringstream ostreamSha1;
 731    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 732       ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 733                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 734    string const ChecksumsSha1 = ostreamSha1.str();
 735
 736    std::ostringstream ostreamSha256;
 737    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 738       ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 739                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 740    string const ChecksumsSha256 = ostreamSha256.str();
 741
 742    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 743    string NewFileName;
 744    if (DirStrip.empty() == false &&
 745        FileName.length() > DirStrip.length() &&
 746        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 747       NewFileName = string(OriginalPath + DirStrip.length());
 748    else
 749       NewFileName = OriginalPath;
 750    if (PathPrefix.empty() == false)
 751       NewFileName = flCombine(PathPrefix,NewFileName);
 752
 753    string Directory = flNotFile(OriginalPath);
 754    string Package = Tags.FindS("Source");
 755
 756    // Perform the delinking operation over all of the files
 757    string ParseJnk;
 758    const char *C = Files.c_str();
 759    char *RealPath = NULL;
 760    for (;isspace(*C); C++);
 761    while (*C != 0)
 762    {
 763       // Parse each of the elements
 764       if (ParseQuoteWord(C,ParseJnk) == false ||
 765           ParseQuoteWord(C,ParseJnk) == false ||
 766           ParseQuoteWord(C,ParseJnk) == false)
 767          return _error->Error("Error parsing file record");
 768
 769       char Jnk[2];
 770       string OriginalPath = Directory + ParseJnk;
 771       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 772           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 773       {
 774          string RP = RealPath;
 775          free(RealPath);
 776          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 777             return false;
 778       }
 779    }
 780
 781    Directory = flNotFile(NewFileName);
 782    if (Directory.length() > 2)
 783       Directory.erase(Directory.end()-1);
 784
 785    // This lists all the changes to the fields we are going to make.
 786    // (5 hardcoded + checksums + maintainer + end marker)
 787    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 788
 789    unsigned int End = 0;
 790    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 791    if (Files.empty() == false)
 792       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 793    if (ChecksumsSha1.empty() == false)
 794       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 795    if (ChecksumsSha256.empty() == false)
 796       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 797    if (Directory != "./")
 798       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 799    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 800    SetTFRewriteData(Changes[End++],"Status",0);
 801
 802    // Rewrite the maintainer field if necessary
 803    bool MaintFailed;
 804    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 805    if (MaintFailed == true)
 806    {
 807       if (NoOverride == false)
 808       {
 809          NewLine(1);
 810          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 811                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 812       }
 813    }
 814    if (NewMaint.empty() == false)
 815       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 816
 817    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 818         I != SOverItem->FieldOverride.end(); I++)
 819       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 820
 821    SetTFRewriteData(Changes[End++], 0, 0);
 822
 823    // Rewrite and store the fields.
 824    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 825       return false;
 826    fprintf(Output,"\n");
 827
 828    Stats.Packages++;
 829
 830    return true;
 831 }
 832                                                                         /*}}}*/
 833
 834 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 835 // ---------------------------------------------------------------------
 836 /* */
 837 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 838                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 839
 840 {
 841    SetExts(".deb");
 842    Output = stdout;
 843 }
 844                                                                         /*}}}*/
 845 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 846 // ---------------------------------------------------------------------
 847 /* If Package is the empty string the control record will be parsed to
 848    determine what the package name is. */
 849 bool ContentsWriter::DoPackage(string FileName, string Package)
 850 {
 851    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 852    {
 853       return false;
 854    }
 855
 856    // Parse the package name
 857    if (Package.empty() == true)
 858    {
 859       Package = Db.Control.Section.FindS("Package");
 860    }
 861
 862    Db.Contents.Add(Gen,Package);
 863
 864    return Db.Finish();
 865 }
 866                                                                         /*}}}*/
 867 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 868 // ---------------------------------------------------------------------
 869 /* */
 870 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 871 {
 872    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 873    if (_error->PendingError() == true)
 874       return false;
 875
 876    // Open the package file
 877    int CompFd = -1;
 878    pid_t Proc = -1;
 879    if (Pkgs.OpenOld(CompFd,Proc) == false)
 880       return false;
 881
 882    // No auto-close FD
 883    FileFd Fd(CompFd,false);
 884    pkgTagFile Tags(&Fd);
 885    if (_error->PendingError() == true)
 886    {
 887       Pkgs.CloseOld(CompFd,Proc);
 888       return false;
 889    }
 890
 891    // Parse.
 892    pkgTagSection Section;
 893    while (Tags.Step(Section) == true)
 894    {
 895       string File = flCombine(Prefix,Section.FindS("FileName"));
 896       string Package = Section.FindS("Section");
 897       if (Package.empty() == false && Package.end()[-1] != '/')
 898       {
 899          Package += '/';
 900          Package += Section.FindS("Package");
 901       }
 902       else
 903          Package += Section.FindS("Package");
 904
 905       DoPackage(File,Package);
 906       if (_error->empty() == false)
 907       {
 908          _error->Error("Errors apply to file '%s'",File.c_str());
 909          _error->DumpErrors();
 910       }
 911    }
 912
 913    // Tidy the compressor
 914    if (Pkgs.CloseOld(CompFd,Proc) == false)
 915       return false;
 916
 917    return true;
 918 }
 919
 920                                                                         /*}}}*/
 921
 922 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 923 // ---------------------------------------------------------------------
 924 /* */
 925 ReleaseWriter::ReleaseWriter(string const &DB)
 926 {
 927    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 928    {
 929       AddPattern("Packages");
 930       AddPattern("Packages.gz");
 931       AddPattern("Packages.bz2");
 932       AddPattern("Packages.lzma");
 933       AddPattern("Packages.xz");
 934       AddPattern("Sources");
 935       AddPattern("Sources.gz");
 936       AddPattern("Sources.bz2");
 937       AddPattern("Sources.lzma");
 938       AddPattern("Sources.xz");
 939       AddPattern("Release");
 940       AddPattern("Index");
 941       AddPattern("md5sum.txt");
 942    }
 943    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 944
 945    Output = stdout;
 946    time_t const now = time(NULL);
 947
 948    setlocale(LC_TIME, "C");
 949
 950    char datestr[128];
 951    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 952                 gmtime(&now)) == 0)
 953    {
 954       datestr[0] = '\0';
 955    }
 956
 957    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 958    char validstr[128];
 959    if (now == validuntil ||
 960        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 961                 gmtime(&validuntil)) == 0)
 962    {
 963       validstr[0] = '\0';
 964    }
 965
 966    setlocale(LC_TIME, "");
 967
 968    map<string,string> Fields;
 969    Fields["Origin"] = "";
 970    Fields["Label"] = "";
 971    Fields["Suite"] = "";
 972    Fields["Version"] = "";
 973    Fields["Codename"] = "";
 974    Fields["Date"] = datestr;
 975    Fields["Valid-Until"] = validstr;
 976    Fields["Architectures"] = "";
 977    Fields["Components"] = "";
 978    Fields["Description"] = "";
 979
 980    for(map<string,string>::const_iterator I = Fields.begin();
 981        I != Fields.end();
 982        ++I)
 983    {
 984       string Config = string("APT::FTPArchive::Release::") + (*I).first;
 985       string Value = _config->Find(Config, (*I).second.c_str());
 986       if (Value == "")
 987          continue;
 988
 989       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
 990    }
 991
 992    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
 993    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
 994    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
 995 }
 996                                                                         /*}}}*/
 997 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
 998 // ---------------------------------------------------------------------
 999 bool ReleaseWriter::DoPackage(string FileName)
1000 {
1001    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1002    string NewFileName;
1003    if (DirStrip.empty() == false &&
1004        FileName.length() > DirStrip.length() &&
1005        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1006                  DirStrip.begin(),DirStrip.end()) == 0)
1007    {
1008       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1009       while (NewFileName[0] == '/')
1010          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1011    }
1012    else
1013       NewFileName = FileName;
1014
1015    if (PathPrefix.empty() == false)
1016       NewFileName = flCombine(PathPrefix,NewFileName);
1017
1018    FileFd fd(FileName, FileFd::ReadOnly);
1019
1020    if (!fd.IsOpen())
1021    {
1022       return false;
1023    }
1024
1025    CheckSums[NewFileName].size = fd.Size();
1026
1027    if (DoMD5 == true)
1028    {
1029       MD5Summation MD5;
1030       MD5.AddFD(fd.Fd(), fd.Size());
1031       CheckSums[NewFileName].MD5 = MD5.Result();
1032       fd.Seek(0);
1033    }
1034    if (DoSHA1 == true)
1035    {
1036       SHA1Summation SHA1;
1037       SHA1.AddFD(fd.Fd(), fd.Size());
1038       CheckSums[NewFileName].SHA1 = SHA1.Result();
1039       fd.Seek(0);
1040    }
1041    if (DoSHA256 == true)
1042    {
1043       SHA256Summation SHA256;
1044       SHA256.AddFD(fd.Fd(), fd.Size());
1045       CheckSums[NewFileName].SHA256 = SHA256.Result();
1046    }
1047
1048    fd.Close();
1049
1050    return true;
1051 }
1052
1053                                                                         /*}}}*/
1054 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1055 // ---------------------------------------------------------------------
1056 void ReleaseWriter::Finish()
1057 {
1058    if (DoMD5 == true)
1059    {
1060       fprintf(Output, "MD5Sum:\n");
1061       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1062           I != CheckSums.end(); ++I)
1063       {
1064          fprintf(Output, " %s %16ld %s\n",
1065                  (*I).second.MD5.c_str(),
1066                  (*I).second.size,
1067                  (*I).first.c_str());
1068       }
1069    }
1070    if (DoSHA1 == true)
1071    {
1072       fprintf(Output, "SHA1:\n");
1073       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1074           I != CheckSums.end(); ++I)
1075       {
1076          fprintf(Output, " %s %16ld %s\n",
1077                  (*I).second.SHA1.c_str(),
1078                  (*I).second.size,
1079                  (*I).first.c_str());
1080       }
1081    }
1082    if (DoSHA256 == true)
1083    {
1084       fprintf(Output, "SHA256:\n");
1085       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1086           I != CheckSums.end(); ++I)
1087       {
1088          fprintf(Output, " %s %16ld %s\n",
1089                  (*I).second.SHA256.c_str(),
1090                  (*I).second.size,
1091                  (*I).first.c_str());
1092       }
1093    }
1094 }