ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include "writer.h"
  15
  16 #include <apti18n.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/md5.h>
  22 #include <apt-pkg/sha1.h>
  23 #include <apt-pkg/sha256.h>
  24 #include <apt-pkg/deblistparser.h>
  25
  26 #include <sys/types.h>
  27 #include <unistd.h>
  28 #include <ctime>
  29 #include <ftw.h>
  30 #include <fnmatch.h>
  31 #include <iostream>
  32 #include <sstream>
  33 #include <memory>
  34
  35 #include "cachedb.h"
  36 #include "apt-ftparchive.h"
  37 #include "multicompress.h"
  38                                                                         /*}}}*/
  39 using namespace std;
  40 FTWScanner *FTWScanner::Owner;
  41
  42 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* */
  45 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  46                              const char *tag,
  47                              const char *rewrite,
  48                              const char *newtag = 0)
  49 {
  50     tfrd.Tag = tag;
  51     tfrd.Rewrite = rewrite;
  52     tfrd.NewTag = newtag;
  53 }
  54                                                                         /*}}}*/
  55
  56 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  57 // ---------------------------------------------------------------------
  58 /* */
  59 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  60 {
  61    ErrorPrinted = false;
  62    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  63
  64    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  65    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  66    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  67 }
  68                                                                         /*}}}*/
  69 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  70 // ---------------------------------------------------------------------
  71 /* This is the FTW scanner, it processes each directory element in the
  72    directory tree. */
  73 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  74 {
  75    if (Flag == FTW_DNR)
  76    {
  77       Owner->NewLine(1);
  78       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  79    }
  80    if (Flag == FTW_NS)
  81    {
  82       Owner->NewLine(1);
  83       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  84    }
  85    if (Flag != FTW_F)
  86       return 0;
  87
  88    return ScannerFile(File, true);
  89 }
  90                                                                         /*}}}*/
  91 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  92 // ---------------------------------------------------------------------
  93 /* */
  94 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  95 {
  96    const char *LastComponent = strrchr(File, '/');
  97    char *RealPath = NULL;
  98
  99    if (LastComponent == NULL)
 100       LastComponent = File;
 101    else
 102       LastComponent++;
 103
 104    vector<string>::const_iterator I;
 105    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 106    {
 107       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 108          break;
 109    }
 110    if (I == Owner->Patterns.end())
 111       return 0;
 112
 113    /* Process it. If the file is a link then resolve it into an absolute
 114       name.. This works best if the directory components the scanner are
 115       given are not links themselves. */
 116    char Jnk[2];
 117    Owner->OriginalPath = File;
 118    if (ReadLink &&
 119        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 120        (RealPath = realpath(File,NULL)) != 0)
 121    {
 122       Owner->DoPackage(RealPath);
 123       free(RealPath);
 124    }
 125    else
 126       Owner->DoPackage(File);
 127
 128    if (_error->empty() == false)
 129    {
 130       // Print any errors or warnings found
 131       string Err;
 132       bool SeenPath = false;
 133       while (_error->empty() == false)
 134       {
 135          Owner->NewLine(1);
 136
 137          bool const Type = _error->PopMessage(Err);
 138          if (Type == true)
 139             cerr << _("E: ") << Err << endl;
 140          else
 141             cerr << _("W: ") << Err << endl;
 142
 143          if (Err.find(File) != string::npos)
 144             SeenPath = true;
 145       }
 146
 147       if (SeenPath == false)
 148          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 149       return 0;
 150    }
 151
 152    return 0;
 153 }
 154                                                                         /*}}}*/
 155 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* */
 158 bool FTWScanner::RecursiveScan(string const &Dir)
 159 {
 160    char *RealPath = NULL;
 161    /* If noprefix is set then jam the scan root in, so we don't generate
 162       link followed paths out of control */
 163    if (InternalPrefix.empty() == true)
 164    {
 165       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 166          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 167       InternalPrefix = RealPath;
 168       free(RealPath);
 169    }
 170
 171    // Do recursive directory searching
 172    Owner = this;
 173    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 174
 175    // Error treewalking?
 176    if (Res != 0)
 177    {
 178       if (_error->PendingError() == false)
 179          _error->Errno("ftw",_("Tree walking failed"));
 180       return false;
 181    }
 182
 183    return true;
 184 }
 185                                                                         /*}}}*/
 186 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 187 // ---------------------------------------------------------------------
 188 /* This is an alternative to using FTW to locate files, it reads the list
 189    of files from another file. */
 190 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 191 {
 192    char *RealPath = NULL;
 193    /* If noprefix is set then jam the scan root in, so we don't generate
 194       link followed paths out of control */
 195    if (InternalPrefix.empty() == true)
 196    {
 197       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 198          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 199       InternalPrefix = RealPath;
 200       free(RealPath);
 201    }
 202
 203    Owner = this;
 204    FILE *List = fopen(File.c_str(),"r");
 205    if (List == 0)
 206       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 207
 208    /* We are a tad tricky here.. We prefix the buffer with the directory
 209       name, that way if we need a full path with just use line.. Sneaky and
 210       fully evil. */
 211    char Line[1000];
 212    char *FileStart;
 213    if (Dir.empty() == true || Dir.end()[-1] != '/')
 214       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 215    else
 216       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 217    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 218    {
 219       char *FileName = _strstrip(FileStart);
 220       if (FileName[0] == 0)
 221          continue;
 222
 223       if (FileName[0] != '/')
 224       {
 225          if (FileName != FileStart)
 226             memmove(FileStart,FileName,strlen(FileStart));
 227          FileName = Line;
 228       }
 229
 230 #if 0
 231       struct stat St;
 232       int Flag = FTW_F;
 233       if (stat(FileName,&St) != 0)
 234          Flag = FTW_NS;
 235 #endif
 236
 237       if (ScannerFile(FileName, false) != 0)
 238          break;
 239    }
 240
 241    fclose(List);
 242    return true;
 243 }
 244                                                                         /*}}}*/
 245 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 246 // ---------------------------------------------------------------------
 247 /* */
 248 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 249                         unsigned long &DeLinkBytes,
 250                         off_t const &FileSize)
 251 {
 252    // See if this isn't an internaly prefix'd file name.
 253    if (InternalPrefix.empty() == false &&
 254        InternalPrefix.length() < FileName.length() &&
 255        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 256                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 257    {
 258       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 259       {
 260          // Tidy up the display
 261          if (DeLinkBytes == 0)
 262             cout << endl;
 263
 264          NewLine(1);
 265          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 266                     SizeToStr(FileSize).c_str());
 267          c1out << flush;
 268
 269          if (NoLinkAct == false)
 270          {
 271             char OldLink[400];
 272             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 273                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 274             else
 275             {
 276                if (unlink(OriginalPath) != 0)
 277                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 278                else
 279                {
 280                   if (link(FileName.c_str(),OriginalPath) != 0)
 281                   {
 282                      // Panic! Restore the symlink
 283                      symlink(OldLink,OriginalPath);
 284                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 285                                           FileName.c_str(),
 286                                           OriginalPath);
 287                   }
 288                }
 289             }
 290          }
 291
 292          DeLinkBytes += FileSize;
 293          if (DeLinkBytes/1024 >= DeLinkLimit)
 294             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 295       }
 296
 297       FileName = OriginalPath;
 298    }
 299
 300    return true;
 301 }
 302                                                                         /*}}}*/
 303
 304 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 305 // ---------------------------------------------------------------------
 306 /* */
 307 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 308                                string const &Arch) :
 309    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 310 {
 311    Output = stdout;
 312    SetExts(".deb .udeb");
 313    DeLinkLimit = 0;
 314
 315    // Process the command line options
 316    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 317    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 318    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 319    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 320    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 321    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 322    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 323
 324    if (Db.Loaded() == false)
 325       DoContents = false;
 326
 327    // Read the override file
 328    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 329       return;
 330    else
 331       NoOverride = true;
 332
 333    if (ExtOverrides.empty() == false)
 334       Over.ReadExtraOverride(ExtOverrides);
 335
 336    _error->DumpErrors();
 337 }
 338                                                                         /*}}}*/
 339 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 340 // ---------------------------------------------------------------------
 341 /* */
 342 bool FTWScanner::SetExts(string const &Vals)
 343 {
 344    ClearPatterns();
 345    string::size_type Start = 0;
 346    while (Start <= Vals.length()-1)
 347    {
 348       string::size_type const Space = Vals.find(' ',Start);
 349       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 350       if ( Arch.empty() == false )
 351       {
 352          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 353          AddPattern(string("*_all") + Vals.substr(Start, Length));
 354       }
 355       else
 356          AddPattern(string("*") + Vals.substr(Start, Length));
 357
 358       Start += Length + 1;
 359    }
 360
 361    return true;
 362 }
 363
 364                                                                         /*}}}*/
 365 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 366 // ---------------------------------------------------------------------
 367 /* This method takes a package and gets its control information and
 368    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 369    rewritten and the path/size/hash appended. */
 370 bool PackagesWriter::DoPackage(string FileName)
 371 {
 372    // Pull all the data we need form the DB
 373    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoAlwaysStat)
 374                   == false)
 375    {
 376       return false;
 377    }
 378
 379    off_t FileSize = Db.GetFileSize();
 380    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 381       return false;
 382
 383    // Lookup the overide information
 384    pkgTagSection &Tags = Db.Control.Section;
 385    string Package = Tags.FindS("Package");
 386    string Architecture;
 387    // if we generate a Packages file for a given arch, we use it to
 388    // look for overrides. if we run in "simple" mode without the
 389    // "Architecures" variable in the config we use the architecure value
 390    // from the deb file
 391    if(Arch != "")
 392       Architecture = Arch;
 393    else
 394       Architecture = Tags.FindS("Architecture");
 395    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 396
 397    if (Package.empty() == true)
 398       return _error->Error(_("Archive had no package field"));
 399
 400    // If we need to do any rewriting of the header do it now..
 401    if (OverItem.get() == 0)
 402    {
 403       if (NoOverride == false)
 404       {
 405          NewLine(1);
 406          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 407       }
 408
 409       OverItem = auto_ptr<Override::Item>(new Override::Item);
 410       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 411       OverItem->Priority = Tags.FindS("Priority");
 412    }
 413
 414    char Size[40];
 415    sprintf(Size,"%lu", (unsigned long) FileSize);
 416
 417    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 418    string NewFileName;
 419    if (DirStrip.empty() == false &&
 420        FileName.length() > DirStrip.length() &&
 421        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 422                  DirStrip.begin(),DirStrip.end()) == 0)
 423       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 424    else
 425       NewFileName = FileName;
 426    if (PathPrefix.empty() == false)
 427       NewFileName = flCombine(PathPrefix,NewFileName);
 428
 429    /* Configuration says we don't want to include the long Description
 430       in the package file - instead we want to ship a separated file */
 431    string desc;
 432    if (LongDescription == false) {
 433       desc = Tags.FindS("Description").append("\n");
 434       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 435    }
 436
 437    // This lists all the changes to the fields we are going to make.
 438    // (7 hardcoded + maintainer + suggests + end marker)
 439    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 440
 441    unsigned int End = 0;
 442    SetTFRewriteData(Changes[End++], "Size", Size);
 443    if (DoMD5 == true)
 444       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 445    if (DoSHA1 == true)
 446       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 447    if (DoSHA256 == true)
 448       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 449    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 450    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 451    SetTFRewriteData(Changes[End++], "Status", 0);
 452    SetTFRewriteData(Changes[End++], "Optional", 0);
 453
 454    string DescriptionMd5;
 455    if (LongDescription == false) {
 456       MD5Summation descmd5;
 457       descmd5.Add(desc.c_str());
 458       DescriptionMd5 = descmd5.Result().Value();
 459       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 460       if (TransWriter != NULL)
 461          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 462    }
 463
 464    // Rewrite the maintainer field if necessary
 465    bool MaintFailed;
 466    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 467    if (MaintFailed == true)
 468    {
 469       if (NoOverride == false)
 470       {
 471          NewLine(1);
 472          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 473                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 474       }
 475    }
 476
 477    if (NewMaint.empty() == false)
 478       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 479
 480    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 481       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 482       but dpkg does this append bit. So we do the append bit, at least that way the
 483       status file and package file will remain similar. There are other transforms
 484       but optional is the only legacy one still in use for some lazy reason. */
 485    string OptionalStr = Tags.FindS("Optional");
 486    if (OptionalStr.empty() == false)
 487    {
 488       if (Tags.FindS("Suggests").empty() == false)
 489          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 490       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 491    }
 492
 493    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 494         I != OverItem->FieldOverride.end(); ++I)
 495       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 496
 497    SetTFRewriteData(Changes[End++], 0, 0);
 498
 499    // Rewrite and store the fields.
 500    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 501       return false;
 502    fprintf(Output,"\n");
 503
 504    return Db.Finish();
 505 }
 506                                                                         /*}}}*/
 507
 508 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 509 // ---------------------------------------------------------------------
 510 /* Create a Translation-Master file for this Packages file */
 511 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 512                                         mode_t const &Permissions) : Output(NULL),
 513                                                         RefCounter(0)
 514 {
 515    if (File.empty() == true)
 516       return;
 517
 518    Comp = new MultiCompress(File, TransCompress, Permissions);
 519    Output = Comp->Input;
 520 }
 521                                                                         /*}}}*/
 522 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 523 // ---------------------------------------------------------------------
 524 /* Create a Translation-Master file for this Packages file */
 525 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 526                                   string const &MD5)
 527 {
 528    if (Output == NULL)
 529       return true;
 530
 531    // Different archs can include different versions and therefore
 532    // different descriptions - so we need to check for both name and md5.
 533    string const Record = Pkg + ":" + MD5;
 534
 535    if (Included.find(Record) != Included.end())
 536       return true;
 537
 538    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 539            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 540
 541    Included.insert(Record);
 542    return true;
 543 }
 544                                                                         /*}}}*/
 545 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 546 // ---------------------------------------------------------------------
 547 /* */
 548 TranslationWriter::~TranslationWriter()
 549 {
 550    if (Comp == NULL)
 551       return;
 552
 553    delete Comp;
 554 }
 555                                                                         /*}}}*/
 556
 557 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 558 // ---------------------------------------------------------------------
 559 /* */
 560 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 561                              string const &ExtOverrides)
 562 {
 563    Output = stdout;
 564    AddPattern("*.dsc");
 565    DeLinkLimit = 0;
 566    Buffer = 0;
 567    BufSize = 0;
 568
 569    // Process the command line options
 570    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 571    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 572    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 573    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 574
 575    // Read the override file
 576    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 577       return;
 578    else
 579       NoOverride = true;
 580
 581    // WTF?? The logic above: if we can't read binary overrides, don't even try
 582    // reading source overrides. if we can read binary overrides, then say there
 583    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 584
 585    if (ExtOverrides.empty() == false)
 586       SOver.ReadExtraOverride(ExtOverrides);
 587
 588    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 589       SOver.ReadOverride(SOverrides,true);
 590 }
 591                                                                         /*}}}*/
 592 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 593 // ---------------------------------------------------------------------
 594 /* */
 595 bool SourcesWriter::DoPackage(string FileName)
 596 {
 597    // Open the archive
 598    FileFd F(FileName,FileFd::ReadOnly);
 599    if (_error->PendingError() == true)
 600       return false;
 601
 602    // Stat the file for later
 603    struct stat St;
 604    if (fstat(F.Fd(),&St) != 0)
 605       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 606
 607    if (St.st_size > 128*1024)
 608       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 609
 610    if (BufSize < (unsigned)St.st_size+1)
 611    {
 612       BufSize = St.st_size+1;
 613       Buffer = (char *)realloc(Buffer,St.st_size+1);
 614    }
 615
 616    if (F.Read(Buffer,St.st_size) == false)
 617       return false;
 618
 619    // Hash the file
 620    char *Start = Buffer;
 621    char *BlkEnd = Buffer + St.st_size;
 622
 623    MD5Summation MD5;
 624    SHA1Summation SHA1;
 625    SHA256Summation SHA256;
 626
 627    if (DoMD5 == true)
 628       MD5.Add((unsigned char *)Start,BlkEnd - Start);
 629    if (DoSHA1 == true)
 630       SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 631    if (DoSHA256 == true)
 632       SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 633
 634    // Add an extra \n to the end, just in case
 635    *BlkEnd++ = '\n';
 636
 637    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 638       before */
 639    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 640    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 641    {
 642       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 643       {
 644          MsgEnd[1] = '\n';
 645          break;
 646       }
 647    }
 648
 649    /* Read records until we locate the Source record. This neatly skips the
 650       GPG header (which is RFC822 formed) without any trouble. */
 651    pkgTagSection Tags;
 652    do
 653    {
 654       unsigned Pos;
 655       if (Tags.Scan(Start,BlkEnd - Start) == false)
 656          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 657       if (Tags.Find("Source",Pos) == true)
 658          break;
 659       Start += Tags.size();
 660    }
 661    while (1);
 662    Tags.Trim();
 663
 664    // Lookup the overide information, finding first the best priority.
 665    string BestPrio;
 666    string Bins = Tags.FindS("Binary");
 667    char Buffer[Bins.length() + 1];
 668    auto_ptr<Override::Item> OverItem(0);
 669    if (Bins.empty() == false)
 670    {
 671       strcpy(Buffer,Bins.c_str());
 672
 673       // Ignore too-long errors.
 674       char *BinList[400];
 675       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 676
 677       // Look at all the binaries
 678       unsigned char BestPrioV = pkgCache::State::Extra;
 679       for (unsigned I = 0; BinList[I] != 0; I++)
 680       {
 681          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 682          if (Itm.get() == 0)
 683             continue;
 684
 685          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 686          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 687          {
 688             BestPrioV = NewPrioV;
 689             BestPrio = Itm->Priority;
 690          }
 691
 692          if (OverItem.get() == 0)
 693             OverItem = Itm;
 694       }
 695    }
 696
 697    // If we need to do any rewriting of the header do it now..
 698    if (OverItem.get() == 0)
 699    {
 700       if (NoOverride == false)
 701       {
 702          NewLine(1);
 703          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 704       }
 705
 706       OverItem = auto_ptr<Override::Item>(new Override::Item);
 707    }
 708
 709    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 710    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 711    if (SOverItem.get() == 0)
 712    {
 713       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 714       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 715       if (SOverItem.get() == 0)
 716       {
 717         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 718          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 719          *SOverItem = *OverItem;
 720       }
 721    }
 722
 723    // Add the dsc to the files hash list
 724    string const strippedName = flNotDir(FileName);
 725    std::ostringstream ostreamFiles;
 726    if (DoMD5 == true && Tags.Exists("Files"))
 727       ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 728                    << strippedName << "\n " << Tags.FindS("Files");
 729    string const Files = ostreamFiles.str();
 730
 731    std::ostringstream ostreamSha1;
 732    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 733       ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 734                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 735    string const ChecksumsSha1 = ostreamSha1.str();
 736
 737    std::ostringstream ostreamSha256;
 738    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 739       ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 740                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 741    string const ChecksumsSha256 = ostreamSha256.str();
 742
 743    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 744    string NewFileName;
 745    if (DirStrip.empty() == false &&
 746        FileName.length() > DirStrip.length() &&
 747        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 748       NewFileName = string(OriginalPath + DirStrip.length());
 749    else
 750       NewFileName = OriginalPath;
 751    if (PathPrefix.empty() == false)
 752       NewFileName = flCombine(PathPrefix,NewFileName);
 753
 754    string Directory = flNotFile(OriginalPath);
 755    string Package = Tags.FindS("Source");
 756
 757    // Perform the delinking operation over all of the files
 758    string ParseJnk;
 759    const char *C = Files.c_str();
 760    char *RealPath = NULL;
 761    for (;isspace(*C); C++);
 762    while (*C != 0)
 763    {
 764       // Parse each of the elements
 765       if (ParseQuoteWord(C,ParseJnk) == false ||
 766           ParseQuoteWord(C,ParseJnk) == false ||
 767           ParseQuoteWord(C,ParseJnk) == false)
 768          return _error->Error("Error parsing file record");
 769
 770       char Jnk[2];
 771       string OriginalPath = Directory + ParseJnk;
 772       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 773           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 774       {
 775          string RP = RealPath;
 776          free(RealPath);
 777          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 778             return false;
 779       }
 780    }
 781
 782    Directory = flNotFile(NewFileName);
 783    if (Directory.length() > 2)
 784       Directory.erase(Directory.end()-1);
 785
 786    // This lists all the changes to the fields we are going to make.
 787    // (5 hardcoded + checksums + maintainer + end marker)
 788    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 789
 790    unsigned int End = 0;
 791    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 792    if (Files.empty() == false)
 793       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 794    if (ChecksumsSha1.empty() == false)
 795       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 796    if (ChecksumsSha256.empty() == false)
 797       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 798    if (Directory != "./")
 799       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 800    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 801    SetTFRewriteData(Changes[End++],"Status",0);
 802
 803    // Rewrite the maintainer field if necessary
 804    bool MaintFailed;
 805    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 806    if (MaintFailed == true)
 807    {
 808       if (NoOverride == false)
 809       {
 810          NewLine(1);
 811          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 812                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 813       }
 814    }
 815    if (NewMaint.empty() == false)
 816       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 817
 818    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 819         I != SOverItem->FieldOverride.end(); ++I)
 820       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 821
 822    SetTFRewriteData(Changes[End++], 0, 0);
 823
 824    // Rewrite and store the fields.
 825    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 826       return false;
 827    fprintf(Output,"\n");
 828
 829    Stats.Packages++;
 830
 831    return true;
 832 }
 833                                                                         /*}}}*/
 834
 835 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 836 // ---------------------------------------------------------------------
 837 /* */
 838 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 839                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 840
 841 {
 842    SetExts(".deb");
 843    Output = stdout;
 844 }
 845                                                                         /*}}}*/
 846 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 847 // ---------------------------------------------------------------------
 848 /* If Package is the empty string the control record will be parsed to
 849    determine what the package name is. */
 850 bool ContentsWriter::DoPackage(string FileName, string Package)
 851 {
 852    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 853    {
 854       return false;
 855    }
 856
 857    // Parse the package name
 858    if (Package.empty() == true)
 859    {
 860       Package = Db.Control.Section.FindS("Package");
 861    }
 862
 863    Db.Contents.Add(Gen,Package);
 864
 865    return Db.Finish();
 866 }
 867                                                                         /*}}}*/
 868 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 869 // ---------------------------------------------------------------------
 870 /* */
 871 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 872 {
 873    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 874    if (_error->PendingError() == true)
 875       return false;
 876
 877    // Open the package file
 878    int CompFd = -1;
 879    pid_t Proc = -1;
 880    if (Pkgs.OpenOld(CompFd,Proc) == false)
 881       return false;
 882
 883    // No auto-close FD
 884    FileFd Fd(CompFd,false);
 885    pkgTagFile Tags(&Fd);
 886    if (_error->PendingError() == true)
 887    {
 888       Pkgs.CloseOld(CompFd,Proc);
 889       return false;
 890    }
 891
 892    // Parse.
 893    pkgTagSection Section;
 894    while (Tags.Step(Section) == true)
 895    {
 896       string File = flCombine(Prefix,Section.FindS("FileName"));
 897       string Package = Section.FindS("Section");
 898       if (Package.empty() == false && Package.end()[-1] != '/')
 899       {
 900          Package += '/';
 901          Package += Section.FindS("Package");
 902       }
 903       else
 904          Package += Section.FindS("Package");
 905
 906       DoPackage(File,Package);
 907       if (_error->empty() == false)
 908       {
 909          _error->Error("Errors apply to file '%s'",File.c_str());
 910          _error->DumpErrors();
 911       }
 912    }
 913
 914    // Tidy the compressor
 915    if (Pkgs.CloseOld(CompFd,Proc) == false)
 916       return false;
 917
 918    return true;
 919 }
 920
 921                                                                         /*}}}*/
 922
 923 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 924 // ---------------------------------------------------------------------
 925 /* */
 926 ReleaseWriter::ReleaseWriter(string const &DB)
 927 {
 928    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 929    {
 930       AddPattern("Packages");
 931       AddPattern("Packages.gz");
 932       AddPattern("Packages.bz2");
 933       AddPattern("Packages.lzma");
 934       AddPattern("Packages.xz");
 935       AddPattern("Sources");
 936       AddPattern("Sources.gz");
 937       AddPattern("Sources.bz2");
 938       AddPattern("Sources.lzma");
 939       AddPattern("Sources.xz");
 940       AddPattern("Release");
 941       AddPattern("Index");
 942       AddPattern("md5sum.txt");
 943    }
 944    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 945
 946    Output = stdout;
 947    time_t const now = time(NULL);
 948
 949    setlocale(LC_TIME, "C");
 950
 951    char datestr[128];
 952    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 953                 gmtime(&now)) == 0)
 954    {
 955       datestr[0] = '\0';
 956    }
 957
 958    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 959    char validstr[128];
 960    if (now == validuntil ||
 961        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 962                 gmtime(&validuntil)) == 0)
 963    {
 964       validstr[0] = '\0';
 965    }
 966
 967    setlocale(LC_TIME, "");
 968
 969    map<string,string> Fields;
 970    Fields["Origin"] = "";
 971    Fields["Label"] = "";
 972    Fields["Suite"] = "";
 973    Fields["Version"] = "";
 974    Fields["Codename"] = "";
 975    Fields["Date"] = datestr;
 976    Fields["Valid-Until"] = validstr;
 977    Fields["Architectures"] = "";
 978    Fields["Components"] = "";
 979    Fields["Description"] = "";
 980
 981    for(map<string,string>::const_iterator I = Fields.begin();
 982        I != Fields.end();
 983        ++I)
 984    {
 985       string Config = string("APT::FTPArchive::Release::") + (*I).first;
 986       string Value = _config->Find(Config, (*I).second.c_str());
 987       if (Value == "")
 988          continue;
 989
 990       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
 991    }
 992
 993    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
 994    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
 995    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
 996 }
 997                                                                         /*}}}*/
 998 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
 999 // ---------------------------------------------------------------------
1000 bool ReleaseWriter::DoPackage(string FileName)
1001 {
1002    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1003    string NewFileName;
1004    if (DirStrip.empty() == false &&
1005        FileName.length() > DirStrip.length() &&
1006        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1007                  DirStrip.begin(),DirStrip.end()) == 0)
1008    {
1009       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1010       while (NewFileName[0] == '/')
1011          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1012    }
1013    else
1014       NewFileName = FileName;
1015
1016    if (PathPrefix.empty() == false)
1017       NewFileName = flCombine(PathPrefix,NewFileName);
1018
1019    FileFd fd(FileName, FileFd::ReadOnly);
1020
1021    if (!fd.IsOpen())
1022    {
1023       return false;
1024    }
1025
1026    CheckSums[NewFileName].size = fd.Size();
1027
1028    if (DoMD5 == true)
1029    {
1030       MD5Summation MD5;
1031       MD5.AddFD(fd.Fd(), fd.Size());
1032       CheckSums[NewFileName].MD5 = MD5.Result();
1033       fd.Seek(0);
1034    }
1035    if (DoSHA1 == true)
1036    {
1037       SHA1Summation SHA1;
1038       SHA1.AddFD(fd.Fd(), fd.Size());
1039       CheckSums[NewFileName].SHA1 = SHA1.Result();
1040       fd.Seek(0);
1041    }
1042    if (DoSHA256 == true)
1043    {
1044       SHA256Summation SHA256;
1045       SHA256.AddFD(fd.Fd(), fd.Size());
1046       CheckSums[NewFileName].SHA256 = SHA256.Result();
1047    }
1048
1049    fd.Close();
1050
1051    return true;
1052 }
1053
1054                                                                         /*}}}*/
1055 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1056 // ---------------------------------------------------------------------
1057 void ReleaseWriter::Finish()
1058 {
1059    if (DoMD5 == true)
1060    {
1061       fprintf(Output, "MD5Sum:\n");
1062       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1063           I != CheckSums.end(); ++I)
1064       {
1065          fprintf(Output, " %s %16ld %s\n",
1066                  (*I).second.MD5.c_str(),
1067                  (*I).second.size,
1068                  (*I).first.c_str());
1069       }
1070    }
1071    if (DoSHA1 == true)
1072    {
1073       fprintf(Output, "SHA1:\n");
1074       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1075           I != CheckSums.end(); ++I)
1076       {
1077          fprintf(Output, " %s %16ld %s\n",
1078                  (*I).second.SHA1.c_str(),
1079                  (*I).second.size,
1080                  (*I).first.c_str());
1081       }
1082    }
1083    if (DoSHA256 == true)
1084    {
1085       fprintf(Output, "SHA256:\n");
1086       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1087           I != CheckSums.end(); ++I)
1088       {
1089          fprintf(Output, " %s %16ld %s\n",
1090                  (*I).second.SHA256.c_str(),
1091                  (*I).second.size,
1092                  (*I).first.c_str());
1093       }
1094    }
1095 }