ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include "writer.h"
  15
  16 #include <apti18n.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/sha1.h>
  22 #include <apt-pkg/sha2.h>
  23 #include <apt-pkg/deblistparser.h>
  24
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <ctime>
  28 #include <ftw.h>
  29 #include <fnmatch.h>
  30 #include <iostream>
  31 #include <sstream>
  32 #include <memory>
  33
  34 #include "cachedb.h"
  35 #include "apt-ftparchive.h"
  36 #include "multicompress.h"
  37                                                                         /*}}}*/
  38 using namespace std;
  39 FTWScanner *FTWScanner::Owner;
  40
  41 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  45                              const char *tag,
  46                              const char *rewrite,
  47                              const char *newtag = 0)
  48 {
  49     tfrd.Tag = tag;
  50     tfrd.Rewrite = rewrite;
  51     tfrd.NewTag = newtag;
  52 }
  53                                                                         /*}}}*/
  54
  55 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  59 {
  60    ErrorPrinted = false;
  61    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  62 }
  63                                                                         /*}}}*/
  64 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* This is the FTW scanner, it processes each directory element in the
  67    directory tree. */
  68 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  69 {
  70    if (Flag == FTW_DNR)
  71    {
  72       Owner->NewLine(1);
  73       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  74    }
  75    if (Flag == FTW_NS)
  76    {
  77       Owner->NewLine(1);
  78       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  79    }
  80    if (Flag != FTW_F)
  81       return 0;
  82
  83    return ScannerFile(File, true);
  84 }
  85                                                                         /*}}}*/
  86 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* */
  89 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  90 {
  91    const char *LastComponent = strrchr(File, '/');
  92    char *RealPath = NULL;
  93
  94    if (LastComponent == NULL)
  95       LastComponent = File;
  96    else
  97       LastComponent++;
  98
  99    vector<string>::const_iterator I;
 100    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 101    {
 102       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 103          break;
 104    }
 105    if (I == Owner->Patterns.end())
 106       return 0;
 107
 108    /* Process it. If the file is a link then resolve it into an absolute
 109       name.. This works best if the directory components the scanner are
 110       given are not links themselves. */
 111    char Jnk[2];
 112    Owner->OriginalPath = File;
 113    if (ReadLink &&
 114        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 115        (RealPath = realpath(File,NULL)) != 0)
 116    {
 117       Owner->DoPackage(RealPath);
 118       free(RealPath);
 119    }
 120    else
 121       Owner->DoPackage(File);
 122
 123    if (_error->empty() == false)
 124    {
 125       // Print any errors or warnings found
 126       string Err;
 127       bool SeenPath = false;
 128       while (_error->empty() == false)
 129       {
 130          Owner->NewLine(1);
 131
 132          bool const Type = _error->PopMessage(Err);
 133          if (Type == true)
 134             cerr << _("E: ") << Err << endl;
 135          else
 136             cerr << _("W: ") << Err << endl;
 137
 138          if (Err.find(File) != string::npos)
 139             SeenPath = true;
 140       }
 141
 142       if (SeenPath == false)
 143          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 144       return 0;
 145    }
 146
 147    return 0;
 148 }
 149                                                                         /*}}}*/
 150 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 151 // ---------------------------------------------------------------------
 152 /* */
 153 bool FTWScanner::RecursiveScan(string const &Dir)
 154 {
 155    char *RealPath = NULL;
 156    /* If noprefix is set then jam the scan root in, so we don't generate
 157       link followed paths out of control */
 158    if (InternalPrefix.empty() == true)
 159    {
 160       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 161          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 162       InternalPrefix = RealPath;
 163       free(RealPath);
 164    }
 165
 166    // Do recursive directory searching
 167    Owner = this;
 168    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 169
 170    // Error treewalking?
 171    if (Res != 0)
 172    {
 173       if (_error->PendingError() == false)
 174          _error->Errno("ftw",_("Tree walking failed"));
 175       return false;
 176    }
 177
 178    return true;
 179 }
 180                                                                         /*}}}*/
 181 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 182 // ---------------------------------------------------------------------
 183 /* This is an alternative to using FTW to locate files, it reads the list
 184    of files from another file. */
 185 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 186 {
 187    char *RealPath = NULL;
 188    /* If noprefix is set then jam the scan root in, so we don't generate
 189       link followed paths out of control */
 190    if (InternalPrefix.empty() == true)
 191    {
 192       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 193          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 194       InternalPrefix = RealPath;
 195       free(RealPath);
 196    }
 197
 198    Owner = this;
 199    FILE *List = fopen(File.c_str(),"r");
 200    if (List == 0)
 201       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 202
 203    /* We are a tad tricky here.. We prefix the buffer with the directory
 204       name, that way if we need a full path with just use line.. Sneaky and
 205       fully evil. */
 206    char Line[1000];
 207    char *FileStart;
 208    if (Dir.empty() == true || Dir.end()[-1] != '/')
 209       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 210    else
 211       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 212    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 213    {
 214       char *FileName = _strstrip(FileStart);
 215       if (FileName[0] == 0)
 216          continue;
 217
 218       if (FileName[0] != '/')
 219       {
 220          if (FileName != FileStart)
 221             memmove(FileStart,FileName,strlen(FileStart));
 222          FileName = Line;
 223       }
 224
 225 #if 0
 226       struct stat St;
 227       int Flag = FTW_F;
 228       if (stat(FileName,&St) != 0)
 229          Flag = FTW_NS;
 230 #endif
 231
 232       if (ScannerFile(FileName, false) != 0)
 233          break;
 234    }
 235
 236    fclose(List);
 237    return true;
 238 }
 239                                                                         /*}}}*/
 240 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 241 // ---------------------------------------------------------------------
 242 /* */
 243 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 244                         unsigned long &DeLinkBytes,
 245                         off_t const &FileSize)
 246 {
 247    // See if this isn't an internaly prefix'd file name.
 248    if (InternalPrefix.empty() == false &&
 249        InternalPrefix.length() < FileName.length() &&
 250        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 251                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 252    {
 253       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 254       {
 255          // Tidy up the display
 256          if (DeLinkBytes == 0)
 257             cout << endl;
 258
 259          NewLine(1);
 260          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 261                     SizeToStr(FileSize).c_str());
 262          c1out << flush;
 263
 264          if (NoLinkAct == false)
 265          {
 266             char OldLink[400];
 267             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 268                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 269             else
 270             {
 271                if (unlink(OriginalPath) != 0)
 272                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 273                else
 274                {
 275                   if (link(FileName.c_str(),OriginalPath) != 0)
 276                   {
 277                      // Panic! Restore the symlink
 278                      symlink(OldLink,OriginalPath);
 279                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 280                                           FileName.c_str(),
 281                                           OriginalPath);
 282                   }
 283                }
 284             }
 285          }
 286
 287          DeLinkBytes += FileSize;
 288          if (DeLinkBytes/1024 >= DeLinkLimit)
 289             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 290       }
 291
 292       FileName = OriginalPath;
 293    }
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298
 299 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 300 // ---------------------------------------------------------------------
 301 /* */
 302 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 303                                string const &Arch) :
 304    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 305 {
 306    Output = stdout;
 307    SetExts(".deb .udeb");
 308    DeLinkLimit = 0;
 309
 310    // Process the command line options
 311    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
 312    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
 313    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
 314    DoSHA256 = _config->FindB("APT::FTPArchive::SHA512",true);
 315    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 316    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 317    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 318    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 319
 320    if (Db.Loaded() == false)
 321       DoContents = false;
 322
 323    // Read the override file
 324    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 325       return;
 326    else
 327       NoOverride = true;
 328
 329    if (ExtOverrides.empty() == false)
 330       Over.ReadExtraOverride(ExtOverrides);
 331
 332    _error->DumpErrors();
 333 }
 334                                                                         /*}}}*/
 335 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 336 // ---------------------------------------------------------------------
 337 /* */
 338 bool FTWScanner::SetExts(string const &Vals)
 339 {
 340    ClearPatterns();
 341    string::size_type Start = 0;
 342    while (Start <= Vals.length()-1)
 343    {
 344       string::size_type const Space = Vals.find(' ',Start);
 345       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 346       if ( Arch.empty() == false )
 347       {
 348          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 349          AddPattern(string("*_all") + Vals.substr(Start, Length));
 350       }
 351       else
 352          AddPattern(string("*") + Vals.substr(Start, Length));
 353
 354       Start += Length + 1;
 355    }
 356
 357    return true;
 358 }
 359
 360                                                                         /*}}}*/
 361 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 362 // ---------------------------------------------------------------------
 363 /* This method takes a package and gets its control information and
 364    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 365    rewritten and the path/size/hash appended. */
 366 bool PackagesWriter::DoPackage(string FileName)
 367 {
 368    // Pull all the data we need form the DB
 369    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 370                   == false)
 371    {
 372       return false;
 373    }
 374
 375    off_t FileSize = Db.GetFileSize();
 376    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 377       return false;
 378
 379    // Lookup the overide information
 380    pkgTagSection &Tags = Db.Control.Section;
 381    string Package = Tags.FindS("Package");
 382    string Architecture;
 383    // if we generate a Packages file for a given arch, we use it to
 384    // look for overrides. if we run in "simple" mode without the
 385    // "Architecures" variable in the config we use the architecure value
 386    // from the deb file
 387    if(Arch != "")
 388       Architecture = Arch;
 389    else
 390       Architecture = Tags.FindS("Architecture");
 391    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 392
 393    if (Package.empty() == true)
 394       return _error->Error(_("Archive had no package field"));
 395
 396    // If we need to do any rewriting of the header do it now..
 397    if (OverItem.get() == 0)
 398    {
 399       if (NoOverride == false)
 400       {
 401          NewLine(1);
 402          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 403       }
 404
 405       OverItem = auto_ptr<Override::Item>(new Override::Item);
 406       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 407       OverItem->Priority = Tags.FindS("Priority");
 408    }
 409
 410    char Size[40];
 411    sprintf(Size,"%lu", (unsigned long) FileSize);
 412
 413    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 414    string NewFileName;
 415    if (DirStrip.empty() == false &&
 416        FileName.length() > DirStrip.length() &&
 417        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 418                  DirStrip.begin(),DirStrip.end()) == 0)
 419       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 420    else
 421       NewFileName = FileName;
 422    if (PathPrefix.empty() == false)
 423       NewFileName = flCombine(PathPrefix,NewFileName);
 424
 425    /* Configuration says we don't want to include the long Description
 426       in the package file - instead we want to ship a separated file */
 427    string desc;
 428    if (LongDescription == false) {
 429       desc = Tags.FindS("Description").append("\n");
 430       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 431    }
 432
 433    // This lists all the changes to the fields we are going to make.
 434    // (7 hardcoded + maintainer + suggests + end marker)
 435    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 436
 437    unsigned int End = 0;
 438    SetTFRewriteData(Changes[End++], "Size", Size);
 439    SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 440    SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 441    SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 442    SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 443    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 444    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 445    SetTFRewriteData(Changes[End++], "Status", 0);
 446    SetTFRewriteData(Changes[End++], "Optional", 0);
 447
 448    string DescriptionMd5;
 449    if (LongDescription == false) {
 450       MD5Summation descmd5;
 451       descmd5.Add(desc.c_str());
 452       DescriptionMd5 = descmd5.Result().Value();
 453       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 454       if (TransWriter != NULL)
 455          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 456    }
 457
 458    // Rewrite the maintainer field if necessary
 459    bool MaintFailed;
 460    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 461    if (MaintFailed == true)
 462    {
 463       if (NoOverride == false)
 464       {
 465          NewLine(1);
 466          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 467                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 468       }
 469    }
 470
 471    if (NewMaint.empty() == false)
 472       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 473
 474    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 475       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 476       but dpkg does this append bit. So we do the append bit, at least that way the
 477       status file and package file will remain similar. There are other transforms
 478       but optional is the only legacy one still in use for some lazy reason. */
 479    string OptionalStr = Tags.FindS("Optional");
 480    if (OptionalStr.empty() == false)
 481    {
 482       if (Tags.FindS("Suggests").empty() == false)
 483          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 484       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 485    }
 486
 487    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 488         I != OverItem->FieldOverride.end(); I++)
 489       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 490
 491    SetTFRewriteData(Changes[End++], 0, 0);
 492
 493    // Rewrite and store the fields.
 494    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 495       return false;
 496    fprintf(Output,"\n");
 497
 498    return Db.Finish();
 499 }
 500                                                                         /*}}}*/
 501
 502 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 503 // ---------------------------------------------------------------------
 504 /* Create a Translation-Master file for this Packages file */
 505 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 506                                         mode_t const &Permissions) : Output(NULL),
 507                                                         RefCounter(0)
 508 {
 509    if (File.empty() == true)
 510       return;
 511
 512    Comp = new MultiCompress(File, TransCompress, Permissions);
 513    Output = Comp->Input;
 514 }
 515                                                                         /*}}}*/
 516 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 517 // ---------------------------------------------------------------------
 518 /* Create a Translation-Master file for this Packages file */
 519 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 520                                   string const &MD5)
 521 {
 522    if (Output == NULL)
 523       return true;
 524
 525    // Different archs can include different versions and therefore
 526    // different descriptions - so we need to check for both name and md5.
 527    string const Record = Pkg + ":" + MD5;
 528
 529    if (Included.find(Record) != Included.end())
 530       return true;
 531
 532    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 533            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 534
 535    Included.insert(Record);
 536    return true;
 537 }
 538                                                                         /*}}}*/
 539 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 540 // ---------------------------------------------------------------------
 541 /* */
 542 TranslationWriter::~TranslationWriter()
 543 {
 544    if (Comp == NULL)
 545       return;
 546
 547    delete Comp;
 548 }
 549                                                                         /*}}}*/
 550
 551 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 552 // ---------------------------------------------------------------------
 553 /* */
 554 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 555                              string const &ExtOverrides)
 556 {
 557    Output = stdout;
 558    AddPattern("*.dsc");
 559    DeLinkLimit = 0;
 560    Buffer = 0;
 561    BufSize = 0;
 562
 563    // Process the command line options
 564    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 565
 566    // Read the override file
 567    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 568       return;
 569    else
 570       NoOverride = true;
 571
 572    // WTF?? The logic above: if we can't read binary overrides, don't even try
 573    // reading source overrides. if we can read binary overrides, then say there
 574    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 575
 576    if (ExtOverrides.empty() == false)
 577       SOver.ReadExtraOverride(ExtOverrides);
 578
 579    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 580       SOver.ReadOverride(SOverrides,true);
 581 }
 582                                                                         /*}}}*/
 583 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 584 // ---------------------------------------------------------------------
 585 /* */
 586 bool SourcesWriter::DoPackage(string FileName)
 587 {
 588    // Open the archive
 589    FileFd F(FileName,FileFd::ReadOnly);
 590    if (_error->PendingError() == true)
 591       return false;
 592
 593    // Stat the file for later
 594    struct stat St;
 595    if (fstat(F.Fd(),&St) != 0)
 596       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 597
 598    if (St.st_size > 128*1024)
 599       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 600
 601    if (BufSize < (unsigned)St.st_size+1)
 602    {
 603       BufSize = St.st_size+1;
 604       Buffer = (char *)realloc(Buffer,St.st_size+1);
 605    }
 606
 607    if (F.Read(Buffer,St.st_size) == false)
 608       return false;
 609
 610    // Hash the file
 611    char *Start = Buffer;
 612    char *BlkEnd = Buffer + St.st_size;
 613    MD5Summation MD5;
 614    MD5.Add((unsigned char *)Start,BlkEnd - Start);
 615
 616    SHA1Summation SHA1;
 617    SHA256Summation SHA256;
 618    SHA512Summation SHA512;
 619    SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 620    SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 621    SHA512.Add((unsigned char *)Start,BlkEnd - Start);
 622
 623    // Add an extra \n to the end, just in case
 624    *BlkEnd++ = '\n';
 625
 626    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 627       before */
 628    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 629    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 630    {
 631       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 632       {
 633          MsgEnd[1] = '\n';
 634          break;
 635       }
 636    }
 637
 638    /* Read records until we locate the Source record. This neatly skips the
 639       GPG header (which is RFC822 formed) without any trouble. */
 640    pkgTagSection Tags;
 641    do
 642    {
 643       unsigned Pos;
 644       if (Tags.Scan(Start,BlkEnd - Start) == false)
 645          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 646       if (Tags.Find("Source",Pos) == true)
 647          break;
 648       Start += Tags.size();
 649    }
 650    while (1);
 651    Tags.Trim();
 652
 653    // Lookup the overide information, finding first the best priority.
 654    string BestPrio;
 655    string Bins = Tags.FindS("Binary");
 656    char Buffer[Bins.length() + 1];
 657    auto_ptr<Override::Item> OverItem(0);
 658    if (Bins.empty() == false)
 659    {
 660       strcpy(Buffer,Bins.c_str());
 661
 662       // Ignore too-long errors.
 663       char *BinList[400];
 664       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 665
 666       // Look at all the binaries
 667       unsigned char BestPrioV = pkgCache::State::Extra;
 668       for (unsigned I = 0; BinList[I] != 0; I++)
 669       {
 670          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 671          if (Itm.get() == 0)
 672             continue;
 673
 674          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 675          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 676          {
 677             BestPrioV = NewPrioV;
 678             BestPrio = Itm->Priority;
 679          }
 680
 681          if (OverItem.get() == 0)
 682             OverItem = Itm;
 683       }
 684    }
 685
 686    // If we need to do any rewriting of the header do it now..
 687    if (OverItem.get() == 0)
 688    {
 689       if (NoOverride == false)
 690       {
 691          NewLine(1);
 692          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 693       }
 694
 695       OverItem = auto_ptr<Override::Item>(new Override::Item);
 696    }
 697
 698    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 699    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 700    if (SOverItem.get() == 0)
 701    {
 702       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 703       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 704       if (SOverItem.get() == 0)
 705       {
 706         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 707          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 708          *SOverItem = *OverItem;
 709       }
 710    }
 711
 712    // Add the dsc to the files hash list
 713    string const strippedName = flNotDir(FileName);
 714    std::ostringstream ostreamFiles;
 715    if (Tags.Exists("Files"))
 716       ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 717                    << strippedName << "\n " << Tags.FindS("Files");
 718    string const Files = ostreamFiles.str();
 719
 720    std::ostringstream ostreamSha1;
 721    if (Tags.Exists("Checksums-Sha1"))
 722       ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 723                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 724    string const ChecksumsSha1 = ostreamSha1.str();
 725
 726    std::ostringstream ostreamSha256;
 727    if (Tags.Exists("Checksums-Sha256"))
 728       ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 729                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 730    string const ChecksumsSha256 = ostreamSha256.str();
 731
 732    std::ostringstream ostreamSha512;
 733    if (Tags.Exists("Checksums-Sha512"))
 734       ostreamSha512 << "\n " << string(SHA512.Result()) << " " << St.st_size << " "
 735                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 736    string const ChecksumsSha512 = ostreamSha512.str();
 737
 738    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 739    string NewFileName;
 740    if (DirStrip.empty() == false &&
 741        FileName.length() > DirStrip.length() &&
 742        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 743       NewFileName = string(OriginalPath + DirStrip.length());
 744    else
 745       NewFileName = OriginalPath;
 746    if (PathPrefix.empty() == false)
 747       NewFileName = flCombine(PathPrefix,NewFileName);
 748
 749    string Directory = flNotFile(OriginalPath);
 750    string Package = Tags.FindS("Source");
 751
 752    // Perform the delinking operation over all of the files
 753    string ParseJnk;
 754    const char *C = Files.c_str();
 755    char *RealPath = NULL;
 756    for (;isspace(*C); C++);
 757    while (*C != 0)
 758    {
 759       // Parse each of the elements
 760       if (ParseQuoteWord(C,ParseJnk) == false ||
 761           ParseQuoteWord(C,ParseJnk) == false ||
 762           ParseQuoteWord(C,ParseJnk) == false)
 763          return _error->Error("Error parsing file record");
 764
 765       char Jnk[2];
 766       string OriginalPath = Directory + ParseJnk;
 767       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 768           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 769       {
 770          string RP = RealPath;
 771          free(RealPath);
 772          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 773             return false;
 774       }
 775    }
 776
 777    Directory = flNotFile(NewFileName);
 778    if (Directory.length() > 2)
 779       Directory.erase(Directory.end()-1);
 780
 781    // This lists all the changes to the fields we are going to make.
 782    // (5 hardcoded + checksums + maintainer + end marker)
 783    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 784
 785    unsigned int End = 0;
 786    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 787    SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 788    SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 789    SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 790    SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 791    if (Directory != "./")
 792       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 793    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 794    SetTFRewriteData(Changes[End++],"Status",0);
 795
 796    // Rewrite the maintainer field if necessary
 797    bool MaintFailed;
 798    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 799    if (MaintFailed == true)
 800    {
 801       if (NoOverride == false)
 802       {
 803          NewLine(1);
 804          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 805                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 806       }
 807    }
 808    if (NewMaint.empty() == false)
 809       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 810
 811    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 812         I != SOverItem->FieldOverride.end(); I++)
 813       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 814
 815    SetTFRewriteData(Changes[End++], 0, 0);
 816
 817    // Rewrite and store the fields.
 818    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 819       return false;
 820    fprintf(Output,"\n");
 821
 822    Stats.Packages++;
 823
 824    return true;
 825 }
 826                                                                         /*}}}*/
 827
 828 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 829 // ---------------------------------------------------------------------
 830 /* */
 831 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 832                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 833
 834 {
 835    SetExts(".deb");
 836    Output = stdout;
 837 }
 838                                                                         /*}}}*/
 839 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 840 // ---------------------------------------------------------------------
 841 /* If Package is the empty string the control record will be parsed to
 842    determine what the package name is. */
 843 bool ContentsWriter::DoPackage(string FileName, string Package)
 844 {
 845    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 846    {
 847       return false;
 848    }
 849
 850    // Parse the package name
 851    if (Package.empty() == true)
 852    {
 853       Package = Db.Control.Section.FindS("Package");
 854    }
 855
 856    Db.Contents.Add(Gen,Package);
 857
 858    return Db.Finish();
 859 }
 860                                                                         /*}}}*/
 861 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 862 // ---------------------------------------------------------------------
 863 /* */
 864 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 865 {
 866    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 867    if (_error->PendingError() == true)
 868       return false;
 869
 870    // Open the package file
 871    int CompFd = -1;
 872    pid_t Proc = -1;
 873    if (Pkgs.OpenOld(CompFd,Proc) == false)
 874       return false;
 875
 876    // No auto-close FD
 877    FileFd Fd(CompFd,false);
 878    pkgTagFile Tags(&Fd);
 879    if (_error->PendingError() == true)
 880    {
 881       Pkgs.CloseOld(CompFd,Proc);
 882       return false;
 883    }
 884
 885    // Parse.
 886    pkgTagSection Section;
 887    while (Tags.Step(Section) == true)
 888    {
 889       string File = flCombine(Prefix,Section.FindS("FileName"));
 890       string Package = Section.FindS("Section");
 891       if (Package.empty() == false && Package.end()[-1] != '/')
 892       {
 893          Package += '/';
 894          Package += Section.FindS("Package");
 895       }
 896       else
 897          Package += Section.FindS("Package");
 898
 899       DoPackage(File,Package);
 900       if (_error->empty() == false)
 901       {
 902          _error->Error("Errors apply to file '%s'",File.c_str());
 903          _error->DumpErrors();
 904       }
 905    }
 906
 907    // Tidy the compressor
 908    if (Pkgs.CloseOld(CompFd,Proc) == false)
 909       return false;
 910
 911    return true;
 912 }
 913
 914                                                                         /*}}}*/
 915
 916 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 917 // ---------------------------------------------------------------------
 918 /* */
 919 ReleaseWriter::ReleaseWriter(string const &DB)
 920 {
 921    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 922    {
 923       AddPattern("Packages");
 924       AddPattern("Packages.gz");
 925       AddPattern("Packages.bz2");
 926       AddPattern("Packages.lzma");
 927       AddPattern("Sources");
 928       AddPattern("Sources.gz");
 929       AddPattern("Sources.bz2");
 930       AddPattern("Sources.lzma");
 931       AddPattern("Release");
 932       AddPattern("Index");
 933       AddPattern("md5sum.txt");
 934    }
 935    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 936
 937    Output = stdout;
 938    time_t const now = time(NULL);
 939    char datestr[128];
 940    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 941                 gmtime(&now)) == 0)
 942    {
 943       datestr[0] = '\0';
 944    }
 945
 946    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 947    char validstr[128];
 948    if (now == validuntil ||
 949        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 950                 gmtime(&validuntil)) == 0)
 951    {
 952       validstr[0] = '\0';
 953    }
 954
 955    map<string,string> Fields;
 956    Fields["Origin"] = "";
 957    Fields["Label"] = "";
 958    Fields["Suite"] = "";
 959    Fields["Version"] = "";
 960    Fields["Codename"] = "";
 961    Fields["Date"] = datestr;
 962    Fields["Valid-Until"] = validstr;
 963    Fields["Architectures"] = "";
 964    Fields["Components"] = "";
 965    Fields["Description"] = "";
 966
 967    for(map<string,string>::const_iterator I = Fields.begin();
 968        I != Fields.end();
 969        ++I)
 970    {
 971       string Config = string("APT::FTPArchive::Release::") + (*I).first;
 972       string Value = _config->Find(Config, (*I).second.c_str());
 973       if (Value == "")
 974          continue;
 975
 976       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
 977    }
 978 }
 979                                                                         /*}}}*/
 980 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
 981 // ---------------------------------------------------------------------
 982 bool ReleaseWriter::DoPackage(string FileName)
 983 {
 984    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 985    string NewFileName;
 986    if (DirStrip.empty() == false &&
 987        FileName.length() > DirStrip.length() &&
 988        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 989                  DirStrip.begin(),DirStrip.end()) == 0)
 990    {
 991       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 992       while (NewFileName[0] == '/')
 993          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
 994    }
 995    else
 996       NewFileName = FileName;
 997
 998    if (PathPrefix.empty() == false)
 999       NewFileName = flCombine(PathPrefix,NewFileName);
1000
1001    FileFd fd(FileName, FileFd::ReadOnly);
1002
1003    if (!fd.IsOpen())
1004    {
1005       return false;
1006    }
1007
1008    CheckSums[NewFileName].size = fd.Size();
1009
1010    MD5Summation MD5;
1011    MD5.AddFD(fd.Fd(), fd.Size());
1012    CheckSums[NewFileName].MD5 = MD5.Result();
1013
1014    fd.Seek(0);
1015    SHA1Summation SHA1;
1016    SHA1.AddFD(fd.Fd(), fd.Size());
1017    CheckSums[NewFileName].SHA1 = SHA1.Result();
1018
1019    fd.Seek(0);
1020    SHA256Summation SHA256;
1021    SHA256.AddFD(fd.Fd(), fd.Size());
1022    CheckSums[NewFileName].SHA256 = SHA256.Result();
1023
1024    SHA256Summation SHA512;
1025    SHA256.AddFD(fd.Fd(), fd.Size());
1026    CheckSums[NewFileName].SHA512 = SHA512.Result();
1027
1028    fd.Close();
1029
1030    return true;
1031 }
1032
1033                                                                         /*}}}*/
1034 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1035 // ---------------------------------------------------------------------
1036 void ReleaseWriter::Finish()
1037 {
1038    fprintf(Output, "MD5Sum:\n");
1039    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1040        I != CheckSums.end();
1041        ++I)
1042    {
1043       fprintf(Output, " %s %16ld %s\n",
1044               (*I).second.MD5.c_str(),
1045               (*I).second.size,
1046               (*I).first.c_str());
1047    }
1048
1049    fprintf(Output, "SHA1:\n");
1050    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1051        I != CheckSums.end();
1052        ++I)
1053    {
1054       fprintf(Output, " %s %16ld %s\n",
1055               (*I).second.SHA1.c_str(),
1056               (*I).second.size,
1057               (*I).first.c_str());
1058    }
1059
1060    fprintf(Output, "SHA256:\n");
1061    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1062        I != CheckSums.end();
1063        ++I)
1064    {
1065       fprintf(Output, " %s %16ld %s\n",
1066               (*I).second.SHA256.c_str(),
1067               (*I).second.size,
1068               (*I).first.c_str());
1069    }
1070
1071    fprintf(Output, "SHA512:\n");
1072    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1073        I != CheckSums.end();
1074        ++I)
1075    {
1076       fprintf(Output, " %s %32ld %s\n",
1077               (*I).second.SHA512.c_str(),
1078               (*I).second.size,
1079               (*I).first.c_str());
1080    }
1081
1082 }
1083