ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include "writer.h"
  15
  16 #include <apti18n.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/sha1.h>
  22 #include <apt-pkg/sha256.h>
  23 #include <apt-pkg/deblistparser.h>
  24
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <ctime>
  28 #include <ftw.h>
  29 #include <fnmatch.h>
  30 #include <iostream>
  31 #include <sstream>
  32 #include <memory>
  33
  34 #include "cachedb.h"
  35 #include "apt-ftparchive.h"
  36 #include "multicompress.h"
  37                                                                         /*}}}*/
  38 using namespace std;
  39 FTWScanner *FTWScanner::Owner;
  40
  41 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  45                              const char *tag,
  46                              const char *rewrite,
  47                              const char *newtag = 0)
  48 {
  49     tfrd.Tag = tag;
  50     tfrd.Rewrite = rewrite;
  51     tfrd.NewTag = newtag;
  52 }
  53                                                                         /*}}}*/
  54
  55 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  59 {
  60    ErrorPrinted = false;
  61    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  62 }
  63                                                                         /*}}}*/
  64 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* This is the FTW scanner, it processes each directory element in the
  67    directory tree. */
  68 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  69 {
  70    if (Flag == FTW_DNR)
  71    {
  72       Owner->NewLine(1);
  73       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  74    }
  75    if (Flag == FTW_NS)
  76    {
  77       Owner->NewLine(1);
  78       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  79    }
  80    if (Flag != FTW_F)
  81       return 0;
  82
  83    return ScannerFile(File, true);
  84 }
  85                                                                         /*}}}*/
  86 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* */
  89 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  90 {
  91    const char *LastComponent = strrchr(File, '/');
  92    char *RealPath = NULL;
  93
  94    if (LastComponent == NULL)
  95       LastComponent = File;
  96    else
  97       LastComponent++;
  98
  99    vector<string>::const_iterator I;
 100    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 101    {
 102       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 103          break;
 104    }
 105    if (I == Owner->Patterns.end())
 106       return 0;
 107
 108    /* Process it. If the file is a link then resolve it into an absolute
 109       name.. This works best if the directory components the scanner are
 110       given are not links themselves. */
 111    char Jnk[2];
 112    Owner->OriginalPath = File;
 113    if (ReadLink &&
 114        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 115        (RealPath = realpath(File,NULL)) != 0)
 116    {
 117       Owner->DoPackage(RealPath);
 118       free(RealPath);
 119    }
 120    else
 121       Owner->DoPackage(File);
 122
 123    if (_error->empty() == false)
 124    {
 125       // Print any errors or warnings found
 126       string Err;
 127       bool SeenPath = false;
 128       while (_error->empty() == false)
 129       {
 130          Owner->NewLine(1);
 131
 132          bool const Type = _error->PopMessage(Err);
 133          if (Type == true)
 134             cerr << _("E: ") << Err << endl;
 135          else
 136             cerr << _("W: ") << Err << endl;
 137
 138          if (Err.find(File) != string::npos)
 139             SeenPath = true;
 140       }
 141
 142       if (SeenPath == false)
 143          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 144       return 0;
 145    }
 146
 147    return 0;
 148 }
 149                                                                         /*}}}*/
 150 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 151 // ---------------------------------------------------------------------
 152 /* */
 153 bool FTWScanner::RecursiveScan(string const &Dir)
 154 {
 155    char *RealPath = NULL;
 156    /* If noprefix is set then jam the scan root in, so we don't generate
 157       link followed paths out of control */
 158    if (InternalPrefix.empty() == true)
 159    {
 160       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 161          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 162       InternalPrefix = RealPath;
 163       free(RealPath);
 164    }
 165
 166    // Do recursive directory searching
 167    Owner = this;
 168    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 169
 170    // Error treewalking?
 171    if (Res != 0)
 172    {
 173       if (_error->PendingError() == false)
 174          _error->Errno("ftw",_("Tree walking failed"));
 175       return false;
 176    }
 177
 178    return true;
 179 }
 180                                                                         /*}}}*/
 181 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 182 // ---------------------------------------------------------------------
 183 /* This is an alternative to using FTW to locate files, it reads the list
 184    of files from another file. */
 185 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 186 {
 187    char *RealPath = NULL;
 188    /* If noprefix is set then jam the scan root in, so we don't generate
 189       link followed paths out of control */
 190    if (InternalPrefix.empty() == true)
 191    {
 192       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 193          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 194       InternalPrefix = RealPath;
 195       free(RealPath);
 196    }
 197
 198    Owner = this;
 199    FILE *List = fopen(File.c_str(),"r");
 200    if (List == 0)
 201       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 202
 203    /* We are a tad tricky here.. We prefix the buffer with the directory
 204       name, that way if we need a full path with just use line.. Sneaky and
 205       fully evil. */
 206    char Line[1000];
 207    char *FileStart;
 208    if (Dir.empty() == true || Dir.end()[-1] != '/')
 209       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 210    else
 211       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 212    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 213    {
 214       char *FileName = _strstrip(FileStart);
 215       if (FileName[0] == 0)
 216          continue;
 217
 218       if (FileName[0] != '/')
 219       {
 220          if (FileName != FileStart)
 221             memmove(FileStart,FileName,strlen(FileStart));
 222          FileName = Line;
 223       }
 224
 225 #if 0
 226       struct stat St;
 227       int Flag = FTW_F;
 228       if (stat(FileName,&St) != 0)
 229          Flag = FTW_NS;
 230 #endif
 231
 232       if (ScannerFile(FileName, false) != 0)
 233          break;
 234    }
 235
 236    fclose(List);
 237    return true;
 238 }
 239                                                                         /*}}}*/
 240 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 241 // ---------------------------------------------------------------------
 242 /* */
 243 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 244                         unsigned long &DeLinkBytes,
 245                         off_t const &FileSize)
 246 {
 247    // See if this isn't an internaly prefix'd file name.
 248    if (InternalPrefix.empty() == false &&
 249        InternalPrefix.length() < FileName.length() &&
 250        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 251                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 252    {
 253       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 254       {
 255          // Tidy up the display
 256          if (DeLinkBytes == 0)
 257             cout << endl;
 258
 259          NewLine(1);
 260          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 261                     SizeToStr(FileSize).c_str());
 262          c1out << flush;
 263
 264          if (NoLinkAct == false)
 265          {
 266             char OldLink[400];
 267             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 268                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 269             else
 270             {
 271                if (unlink(OriginalPath) != 0)
 272                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 273                else
 274                {
 275                   if (link(FileName.c_str(),OriginalPath) != 0)
 276                   {
 277                      // Panic! Restore the symlink
 278                      symlink(OldLink,OriginalPath);
 279                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 280                                           FileName.c_str(),
 281                                           OriginalPath);
 282                   }
 283                }
 284             }
 285          }
 286
 287          DeLinkBytes += FileSize;
 288          if (DeLinkBytes/1024 >= DeLinkLimit)
 289             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 290       }
 291
 292       FileName = OriginalPath;
 293    }
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298
 299 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 300 // ---------------------------------------------------------------------
 301 /* */
 302 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 303                                string const &Arch) :
 304    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 305 {
 306    Output = stdout;
 307    SetExts(".deb .udeb");
 308    DeLinkLimit = 0;
 309
 310    // Process the command line options
 311    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
 312    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
 313    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
 314    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 315    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 316    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 317    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 318
 319    if (Db.Loaded() == false)
 320       DoContents = false;
 321
 322    // Read the override file
 323    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 324       return;
 325    else
 326       NoOverride = true;
 327
 328    if (ExtOverrides.empty() == false)
 329       Over.ReadExtraOverride(ExtOverrides);
 330
 331    _error->DumpErrors();
 332 }
 333                                                                         /*}}}*/
 334 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 335 // ---------------------------------------------------------------------
 336 /* */
 337 bool FTWScanner::SetExts(string const &Vals)
 338 {
 339    ClearPatterns();
 340    string::size_type Start = 0;
 341    while (Start <= Vals.length()-1)
 342    {
 343       string::size_type const Space = Vals.find(' ',Start);
 344       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 345       if ( Arch.empty() == false )
 346       {
 347          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 348          AddPattern(string("*_all") + Vals.substr(Start, Length));
 349       }
 350       else
 351          AddPattern(string("*") + Vals.substr(Start, Length));
 352
 353       Start += Length + 1;
 354    }
 355
 356    return true;
 357 }
 358
 359                                                                         /*}}}*/
 360 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 361 // ---------------------------------------------------------------------
 362 /* This method takes a package and gets its control information and
 363    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 364    rewritten and the path/size/hash appended. */
 365 bool PackagesWriter::DoPackage(string FileName)
 366 {
 367    // Pull all the data we need form the DB
 368    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoAlwaysStat)
 369                   == false)
 370    {
 371       return false;
 372    }
 373
 374    off_t FileSize = Db.GetFileSize();
 375    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 376       return false;
 377
 378    // Lookup the overide information
 379    pkgTagSection &Tags = Db.Control.Section;
 380    string Package = Tags.FindS("Package");
 381    string Architecture;
 382    // if we generate a Packages file for a given arch, we use it to
 383    // look for overrides. if we run in "simple" mode without the
 384    // "Architecures" variable in the config we use the architecure value
 385    // from the deb file
 386    if(Arch != "")
 387       Architecture = Arch;
 388    else
 389       Architecture = Tags.FindS("Architecture");
 390    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 391
 392    if (Package.empty() == true)
 393       return _error->Error(_("Archive had no package field"));
 394
 395    // If we need to do any rewriting of the header do it now..
 396    if (OverItem.get() == 0)
 397    {
 398       if (NoOverride == false)
 399       {
 400          NewLine(1);
 401          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 402       }
 403
 404       OverItem = auto_ptr<Override::Item>(new Override::Item);
 405       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 406       OverItem->Priority = Tags.FindS("Priority");
 407    }
 408
 409    char Size[40];
 410    sprintf(Size,"%lu", (unsigned long) FileSize);
 411
 412    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 413    string NewFileName;
 414    if (DirStrip.empty() == false &&
 415        FileName.length() > DirStrip.length() &&
 416        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 417                  DirStrip.begin(),DirStrip.end()) == 0)
 418       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 419    else
 420       NewFileName = FileName;
 421    if (PathPrefix.empty() == false)
 422       NewFileName = flCombine(PathPrefix,NewFileName);
 423
 424    /* Configuration says we don't want to include the long Description
 425       in the package file - instead we want to ship a separated file */
 426    string desc;
 427    if (LongDescription == false) {
 428       desc = Tags.FindS("Description").append("\n");
 429       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 430    }
 431
 432    // This lists all the changes to the fields we are going to make.
 433    // (7 hardcoded + maintainer + suggests + end marker)
 434    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 435
 436    unsigned int End = 0;
 437    SetTFRewriteData(Changes[End++], "Size", Size);
 438    SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 439    SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 440    SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 441    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 442    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 443    SetTFRewriteData(Changes[End++], "Status", 0);
 444    SetTFRewriteData(Changes[End++], "Optional", 0);
 445
 446    string DescriptionMd5;
 447    if (LongDescription == false) {
 448       MD5Summation descmd5;
 449       descmd5.Add(desc.c_str());
 450       DescriptionMd5 = descmd5.Result().Value();
 451       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 452       if (TransWriter != NULL)
 453          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 454    }
 455
 456    // Rewrite the maintainer field if necessary
 457    bool MaintFailed;
 458    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 459    if (MaintFailed == true)
 460    {
 461       if (NoOverride == false)
 462       {
 463          NewLine(1);
 464          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 465                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 466       }
 467    }
 468
 469    if (NewMaint.empty() == false)
 470       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 471
 472    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 473       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 474       but dpkg does this append bit. So we do the append bit, at least that way the
 475       status file and package file will remain similar. There are other transforms
 476       but optional is the only legacy one still in use for some lazy reason. */
 477    string OptionalStr = Tags.FindS("Optional");
 478    if (OptionalStr.empty() == false)
 479    {
 480       if (Tags.FindS("Suggests").empty() == false)
 481          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 482       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 483    }
 484
 485    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 486         I != OverItem->FieldOverride.end(); I++)
 487       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 488
 489    SetTFRewriteData(Changes[End++], 0, 0);
 490
 491    // Rewrite and store the fields.
 492    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 493       return false;
 494    fprintf(Output,"\n");
 495
 496    return Db.Finish();
 497 }
 498                                                                         /*}}}*/
 499
 500 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 501 // ---------------------------------------------------------------------
 502 /* Create a Translation-Master file for this Packages file */
 503 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 504                                         mode_t const &Permissions) : Output(NULL),
 505                                                         RefCounter(0)
 506 {
 507    if (File.empty() == true)
 508       return;
 509
 510    Comp = new MultiCompress(File, TransCompress, Permissions);
 511    Output = Comp->Input;
 512 }
 513                                                                         /*}}}*/
 514 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 515 // ---------------------------------------------------------------------
 516 /* Create a Translation-Master file for this Packages file */
 517 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 518                                   string const &MD5)
 519 {
 520    if (Output == NULL)
 521       return true;
 522
 523    // Different archs can include different versions and therefore
 524    // different descriptions - so we need to check for both name and md5.
 525    string const Record = Pkg + ":" + MD5;
 526
 527    if (Included.find(Record) != Included.end())
 528       return true;
 529
 530    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 531            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 532
 533    Included.insert(Record);
 534    return true;
 535 }
 536                                                                         /*}}}*/
 537 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 538 // ---------------------------------------------------------------------
 539 /* */
 540 TranslationWriter::~TranslationWriter()
 541 {
 542    if (Comp == NULL)
 543       return;
 544
 545    delete Comp;
 546 }
 547                                                                         /*}}}*/
 548
 549 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 550 // ---------------------------------------------------------------------
 551 /* */
 552 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 553                              string const &ExtOverrides)
 554 {
 555    Output = stdout;
 556    AddPattern("*.dsc");
 557    DeLinkLimit = 0;
 558    Buffer = 0;
 559    BufSize = 0;
 560
 561    // Process the command line options
 562    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 563
 564    // Read the override file
 565    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 566       return;
 567    else
 568       NoOverride = true;
 569
 570    // WTF?? The logic above: if we can't read binary overrides, don't even try
 571    // reading source overrides. if we can read binary overrides, then say there
 572    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 573
 574    if (ExtOverrides.empty() == false)
 575       SOver.ReadExtraOverride(ExtOverrides);
 576
 577    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 578       SOver.ReadOverride(SOverrides,true);
 579 }
 580                                                                         /*}}}*/
 581 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* */
 584 bool SourcesWriter::DoPackage(string FileName)
 585 {
 586    // Open the archive
 587    FileFd F(FileName,FileFd::ReadOnly);
 588    if (_error->PendingError() == true)
 589       return false;
 590
 591    // Stat the file for later
 592    struct stat St;
 593    if (fstat(F.Fd(),&St) != 0)
 594       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 595
 596    if (St.st_size > 128*1024)
 597       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 598
 599    if (BufSize < (unsigned)St.st_size+1)
 600    {
 601       BufSize = St.st_size+1;
 602       Buffer = (char *)realloc(Buffer,St.st_size+1);
 603    }
 604
 605    if (F.Read(Buffer,St.st_size) == false)
 606       return false;
 607
 608    // Hash the file
 609    char *Start = Buffer;
 610    char *BlkEnd = Buffer + St.st_size;
 611    MD5Summation MD5;
 612    MD5.Add((unsigned char *)Start,BlkEnd - Start);
 613
 614    SHA1Summation SHA1;
 615    SHA256Summation SHA256;
 616    SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 617    SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 618
 619    // Add an extra \n to the end, just in case
 620    *BlkEnd++ = '\n';
 621
 622    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 623       before */
 624    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 625    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 626    {
 627       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 628       {
 629          MsgEnd[1] = '\n';
 630          break;
 631       }
 632    }
 633
 634    /* Read records until we locate the Source record. This neatly skips the
 635       GPG header (which is RFC822 formed) without any trouble. */
 636    pkgTagSection Tags;
 637    do
 638    {
 639       unsigned Pos;
 640       if (Tags.Scan(Start,BlkEnd - Start) == false)
 641          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 642       if (Tags.Find("Source",Pos) == true)
 643          break;
 644       Start += Tags.size();
 645    }
 646    while (1);
 647    Tags.Trim();
 648
 649    // Lookup the overide information, finding first the best priority.
 650    string BestPrio;
 651    string Bins = Tags.FindS("Binary");
 652    char Buffer[Bins.length() + 1];
 653    auto_ptr<Override::Item> OverItem(0);
 654    if (Bins.empty() == false)
 655    {
 656       strcpy(Buffer,Bins.c_str());
 657
 658       // Ignore too-long errors.
 659       char *BinList[400];
 660       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 661
 662       // Look at all the binaries
 663       unsigned char BestPrioV = pkgCache::State::Extra;
 664       for (unsigned I = 0; BinList[I] != 0; I++)
 665       {
 666          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 667          if (Itm.get() == 0)
 668             continue;
 669
 670          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 671          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 672          {
 673             BestPrioV = NewPrioV;
 674             BestPrio = Itm->Priority;
 675          }
 676
 677          if (OverItem.get() == 0)
 678             OverItem = Itm;
 679       }
 680    }
 681
 682    // If we need to do any rewriting of the header do it now..
 683    if (OverItem.get() == 0)
 684    {
 685       if (NoOverride == false)
 686       {
 687          NewLine(1);
 688          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 689       }
 690
 691       OverItem = auto_ptr<Override::Item>(new Override::Item);
 692    }
 693
 694    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 695    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 696    if (SOverItem.get() == 0)
 697    {
 698       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 699       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 700       if (SOverItem.get() == 0)
 701       {
 702         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 703          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 704          *SOverItem = *OverItem;
 705       }
 706    }
 707
 708    // Add the dsc to the files hash list
 709    string const strippedName = flNotDir(FileName);
 710    std::ostringstream ostreamFiles;
 711    if (Tags.Exists("Files"))
 712       ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 713                    << strippedName << "\n " << Tags.FindS("Files");
 714    string const Files = ostreamFiles.str();
 715
 716    std::ostringstream ostreamSha1;
 717    if (Tags.Exists("Checksums-Sha1"))
 718       ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 719                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 720    string const ChecksumsSha1 = ostreamSha1.str();
 721
 722    std::ostringstream ostreamSha256;
 723    if (Tags.Exists("Checksums-Sha256"))
 724       ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 725                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 726    string const ChecksumsSha256 = ostreamSha256.str();
 727
 728    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 729    string NewFileName;
 730    if (DirStrip.empty() == false &&
 731        FileName.length() > DirStrip.length() &&
 732        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 733       NewFileName = string(OriginalPath + DirStrip.length());
 734    else
 735       NewFileName = OriginalPath;
 736    if (PathPrefix.empty() == false)
 737       NewFileName = flCombine(PathPrefix,NewFileName);
 738
 739    string Directory = flNotFile(OriginalPath);
 740    string Package = Tags.FindS("Source");
 741
 742    // Perform the delinking operation over all of the files
 743    string ParseJnk;
 744    const char *C = Files.c_str();
 745    char *RealPath = NULL;
 746    for (;isspace(*C); C++);
 747    while (*C != 0)
 748    {
 749       // Parse each of the elements
 750       if (ParseQuoteWord(C,ParseJnk) == false ||
 751           ParseQuoteWord(C,ParseJnk) == false ||
 752           ParseQuoteWord(C,ParseJnk) == false)
 753          return _error->Error("Error parsing file record");
 754
 755       char Jnk[2];
 756       string OriginalPath = Directory + ParseJnk;
 757       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 758           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 759       {
 760          string RP = RealPath;
 761          free(RealPath);
 762          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 763             return false;
 764       }
 765    }
 766
 767    Directory = flNotFile(NewFileName);
 768    if (Directory.length() > 2)
 769       Directory.erase(Directory.end()-1);
 770
 771    // This lists all the changes to the fields we are going to make.
 772    // (5 hardcoded + checksums + maintainer + end marker)
 773    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 774
 775    unsigned int End = 0;
 776    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 777    SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 778    SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 779    SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 780    if (Directory != "./")
 781       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 782    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 783    SetTFRewriteData(Changes[End++],"Status",0);
 784
 785    // Rewrite the maintainer field if necessary
 786    bool MaintFailed;
 787    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 788    if (MaintFailed == true)
 789    {
 790       if (NoOverride == false)
 791       {
 792          NewLine(1);
 793          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 794                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 795       }
 796    }
 797    if (NewMaint.empty() == false)
 798       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 799
 800    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 801         I != SOverItem->FieldOverride.end(); I++)
 802       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 803
 804    SetTFRewriteData(Changes[End++], 0, 0);
 805
 806    // Rewrite and store the fields.
 807    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 808       return false;
 809    fprintf(Output,"\n");
 810
 811    Stats.Packages++;
 812
 813    return true;
 814 }
 815                                                                         /*}}}*/
 816
 817 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 818 // ---------------------------------------------------------------------
 819 /* */
 820 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 821                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 822
 823 {
 824    SetExts(".deb");
 825    Output = stdout;
 826 }
 827                                                                         /*}}}*/
 828 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 829 // ---------------------------------------------------------------------
 830 /* If Package is the empty string the control record will be parsed to
 831    determine what the package name is. */
 832 bool ContentsWriter::DoPackage(string FileName, string Package)
 833 {
 834    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 835    {
 836       return false;
 837    }
 838
 839    // Parse the package name
 840    if (Package.empty() == true)
 841    {
 842       Package = Db.Control.Section.FindS("Package");
 843    }
 844
 845    Db.Contents.Add(Gen,Package);
 846
 847    return Db.Finish();
 848 }
 849                                                                         /*}}}*/
 850 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 851 // ---------------------------------------------------------------------
 852 /* */
 853 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 854 {
 855    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 856    if (_error->PendingError() == true)
 857       return false;
 858
 859    // Open the package file
 860    int CompFd = -1;
 861    pid_t Proc = -1;
 862    if (Pkgs.OpenOld(CompFd,Proc) == false)
 863       return false;
 864
 865    // No auto-close FD
 866    FileFd Fd(CompFd,false);
 867    pkgTagFile Tags(&Fd);
 868    if (_error->PendingError() == true)
 869    {
 870       Pkgs.CloseOld(CompFd,Proc);
 871       return false;
 872    }
 873
 874    // Parse.
 875    pkgTagSection Section;
 876    while (Tags.Step(Section) == true)
 877    {
 878       string File = flCombine(Prefix,Section.FindS("FileName"));
 879       string Package = Section.FindS("Section");
 880       if (Package.empty() == false && Package.end()[-1] != '/')
 881       {
 882          Package += '/';
 883          Package += Section.FindS("Package");
 884       }
 885       else
 886          Package += Section.FindS("Package");
 887
 888       DoPackage(File,Package);
 889       if (_error->empty() == false)
 890       {
 891          _error->Error("Errors apply to file '%s'",File.c_str());
 892          _error->DumpErrors();
 893       }
 894    }
 895
 896    // Tidy the compressor
 897    if (Pkgs.CloseOld(CompFd,Proc) == false)
 898       return false;
 899
 900    return true;
 901 }
 902
 903                                                                         /*}}}*/
 904
 905 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 906 // ---------------------------------------------------------------------
 907 /* */
 908 ReleaseWriter::ReleaseWriter(string const &DB)
 909 {
 910    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 911    {
 912       AddPattern("Packages");
 913       AddPattern("Packages.gz");
 914       AddPattern("Packages.bz2");
 915       AddPattern("Packages.lzma");
 916       AddPattern("Sources");
 917       AddPattern("Sources.gz");
 918       AddPattern("Sources.bz2");
 919       AddPattern("Sources.lzma");
 920       AddPattern("Release");
 921       AddPattern("Index");
 922       AddPattern("md5sum.txt");
 923    }
 924    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 925
 926    Output = stdout;
 927    time_t const now = time(NULL);
 928    char datestr[128];
 929    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 930                 gmtime(&now)) == 0)
 931    {
 932       datestr[0] = '\0';
 933    }
 934
 935    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 936    char validstr[128];
 937    if (now == validuntil ||
 938        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 939                 gmtime(&validuntil)) == 0)
 940    {
 941       validstr[0] = '\0';
 942    }
 943
 944    map<string,string> Fields;
 945    Fields["Origin"] = "";
 946    Fields["Label"] = "";
 947    Fields["Suite"] = "";
 948    Fields["Version"] = "";
 949    Fields["Codename"] = "";
 950    Fields["Date"] = datestr;
 951    Fields["Valid-Until"] = validstr;
 952    Fields["Architectures"] = "";
 953    Fields["Components"] = "";
 954    Fields["Description"] = "";
 955
 956    for(map<string,string>::const_iterator I = Fields.begin();
 957        I != Fields.end();
 958        ++I)
 959    {
 960       string Config = string("APT::FTPArchive::Release::") + (*I).first;
 961       string Value = _config->Find(Config, (*I).second.c_str());
 962       if (Value == "")
 963          continue;
 964
 965       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
 966    }
 967 }
 968                                                                         /*}}}*/
 969 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
 970 // ---------------------------------------------------------------------
 971 bool ReleaseWriter::DoPackage(string FileName)
 972 {
 973    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 974    string NewFileName;
 975    if (DirStrip.empty() == false &&
 976        FileName.length() > DirStrip.length() &&
 977        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 978                  DirStrip.begin(),DirStrip.end()) == 0)
 979    {
 980       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 981       while (NewFileName[0] == '/')
 982          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
 983    }
 984    else
 985       NewFileName = FileName;
 986
 987    if (PathPrefix.empty() == false)
 988       NewFileName = flCombine(PathPrefix,NewFileName);
 989
 990    FileFd fd(FileName, FileFd::ReadOnly);
 991
 992    if (!fd.IsOpen())
 993    {
 994       return false;
 995    }
 996
 997    CheckSums[NewFileName].size = fd.Size();
 998
 999    MD5Summation MD5;
1000    MD5.AddFD(fd.Fd(), fd.Size());
1001    CheckSums[NewFileName].MD5 = MD5.Result();
1002
1003    fd.Seek(0);
1004    SHA1Summation SHA1;
1005    SHA1.AddFD(fd.Fd(), fd.Size());
1006    CheckSums[NewFileName].SHA1 = SHA1.Result();
1007
1008    fd.Seek(0);
1009    SHA256Summation SHA256;
1010    SHA256.AddFD(fd.Fd(), fd.Size());
1011    CheckSums[NewFileName].SHA256 = SHA256.Result();
1012
1013    fd.Close();
1014
1015    return true;
1016 }
1017
1018                                                                         /*}}}*/
1019 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1020 // ---------------------------------------------------------------------
1021 void ReleaseWriter::Finish()
1022 {
1023    fprintf(Output, "MD5Sum:\n");
1024    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1025        I != CheckSums.end();
1026        ++I)
1027    {
1028       fprintf(Output, " %s %16ld %s\n",
1029               (*I).second.MD5.c_str(),
1030               (*I).second.size,
1031               (*I).first.c_str());
1032    }
1033
1034    fprintf(Output, "SHA1:\n");
1035    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1036        I != CheckSums.end();
1037        ++I)
1038    {
1039       fprintf(Output, " %s %16ld %s\n",
1040               (*I).second.SHA1.c_str(),
1041               (*I).second.size,
1042               (*I).first.c_str());
1043    }
1044
1045    fprintf(Output, "SHA256:\n");
1046    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1047        I != CheckSums.end();
1048        ++I)
1049    {
1050       fprintf(Output, " %s %16ld %s\n",
1051               (*I).second.SHA256.c_str(),
1052               (*I).second.size,
1053               (*I).first.c_str());
1054    }
1055 }
1056