ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/deblistparser.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/fileutl.h>
  20 #include <apt-pkg/gpgv.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/debfile.h>
  25 #include <apt-pkg/pkgcache.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/sha2.h>
  28 #include <apt-pkg/tagfile.h>
  29
  30 #include <ctype.h>
  31 #include <fnmatch.h>
  32 #include <ftw.h>
  33 #include <locale.h>
  34 #include <string.h>
  35 #include <sys/stat.h>
  36 #include <sys/types.h>
  37 #include <unistd.h>
  38 #include <ctime>
  39 #include <iostream>
  40 #include <sstream>
  41 #include <memory>
  42 #include <utility>
  43
  44 #include "apt-ftparchive.h"
  45 #include "writer.h"
  46 #include "cachedb.h"
  47 #include "multicompress.h"
  48
  49 #include <apti18n.h>
  50                                                                         /*}}}*/
  51 using namespace std;
  52 FTWScanner *FTWScanner::Owner;
  53
  54 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 static inline TFRewriteData SetTFRewriteData(const char *tag,
  58                              const char *rewrite,
  59                              const char *newtag = 0)
  60 {
  61    TFRewriteData tfrd;
  62    tfrd.Tag = tag;
  63    tfrd.Rewrite = rewrite;
  64    tfrd.NewTag = newtag;
  65    return tfrd;
  66 }
  67                                                                         /*}}}*/
  68
  69 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  70 // ---------------------------------------------------------------------
  71 /* */
  72 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  73 {
  74    ErrorPrinted = false;
  75    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  76
  77    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  78    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  79    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  80    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  81 }
  82                                                                         /*}}}*/
  83 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* This is the FTW scanner, it processes each directory element in the
  86    directory tree. */
  87 int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
  88 {
  89    if (Flag == FTW_DNR)
  90    {
  91       Owner->NewLine(1);
  92       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  93    }
  94    if (Flag == FTW_NS)
  95    {
  96       Owner->NewLine(1);
  97       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  98    }
  99    if (Flag != FTW_F)
 100       return 0;
 101
 102    return ScannerFile(File, true);
 103 }
 104                                                                         /*}}}*/
 105 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
 109 {
 110    const char *LastComponent = strrchr(File, '/');
 111    char *RealPath = NULL;
 112
 113    if (LastComponent == NULL)
 114       LastComponent = File;
 115    else
 116       LastComponent++;
 117
 118    vector<string>::const_iterator I;
 119    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 120    {
 121       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 122          break;
 123    }
 124    if (I == Owner->Patterns.end())
 125       return 0;
 126
 127    /* Process it. If the file is a link then resolve it into an absolute
 128       name.. This works best if the directory components the scanner are
 129       given are not links themselves. */
 130    char Jnk[2];
 131    Owner->OriginalPath = File;
 132    if (ReadLink &&
 133        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 134        (RealPath = realpath(File,NULL)) != 0)
 135    {
 136       Owner->DoPackage(RealPath);
 137       free(RealPath);
 138    }
 139    else
 140       Owner->DoPackage(File);
 141
 142    if (_error->empty() == false)
 143    {
 144       // Print any errors or warnings found
 145       string Err;
 146       bool SeenPath = false;
 147       while (_error->empty() == false)
 148       {
 149          Owner->NewLine(1);
 150
 151          bool const Type = _error->PopMessage(Err);
 152          if (Type == true)
 153             cerr << _("E: ") << Err << endl;
 154          else
 155             cerr << _("W: ") << Err << endl;
 156
 157          if (Err.find(File) != string::npos)
 158             SeenPath = true;
 159       }
 160
 161       if (SeenPath == false)
 162          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 163       return 0;
 164    }
 165
 166    return 0;
 167 }
 168                                                                         /*}}}*/
 169 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 170 // ---------------------------------------------------------------------
 171 /* */
 172 bool FTWScanner::RecursiveScan(string const &Dir)
 173 {
 174    char *RealPath = NULL;
 175    /* If noprefix is set then jam the scan root in, so we don't generate
 176       link followed paths out of control */
 177    if (InternalPrefix.empty() == true)
 178    {
 179       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 180          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 181       InternalPrefix = RealPath;
 182       free(RealPath);
 183    }
 184
 185    // Do recursive directory searching
 186    Owner = this;
 187    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 188
 189    // Error treewalking?
 190    if (Res != 0)
 191    {
 192       if (_error->PendingError() == false)
 193          _error->Errno("ftw",_("Tree walking failed"));
 194       return false;
 195    }
 196
 197    return true;
 198 }
 199                                                                         /*}}}*/
 200 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 201 // ---------------------------------------------------------------------
 202 /* This is an alternative to using FTW to locate files, it reads the list
 203    of files from another file. */
 204 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 205 {
 206    char *RealPath = NULL;
 207    /* If noprefix is set then jam the scan root in, so we don't generate
 208       link followed paths out of control */
 209    if (InternalPrefix.empty() == true)
 210    {
 211       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 212          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 213       InternalPrefix = RealPath;
 214       free(RealPath);
 215    }
 216
 217    Owner = this;
 218    FILE *List = fopen(File.c_str(),"r");
 219    if (List == 0)
 220       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 221
 222    /* We are a tad tricky here.. We prefix the buffer with the directory
 223       name, that way if we need a full path with just use line.. Sneaky and
 224       fully evil. */
 225    char Line[1000];
 226    char *FileStart;
 227    if (Dir.empty() == true || Dir.end()[-1] != '/')
 228       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 229    else
 230       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 231    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 232    {
 233       char *FileName = _strstrip(FileStart);
 234       if (FileName[0] == 0)
 235          continue;
 236
 237       if (FileName[0] != '/')
 238       {
 239          if (FileName != FileStart)
 240             memmove(FileStart,FileName,strlen(FileStart));
 241          FileName = Line;
 242       }
 243
 244 #if 0
 245       struct stat St;
 246       int Flag = FTW_F;
 247       if (stat(FileName,&St) != 0)
 248          Flag = FTW_NS;
 249 #endif
 250
 251       if (ScannerFile(FileName, false) != 0)
 252          break;
 253    }
 254
 255    fclose(List);
 256    return true;
 257 }
 258                                                                         /*}}}*/
 259 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 260 // ---------------------------------------------------------------------
 261 /* */
 262 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 263                         unsigned long long &DeLinkBytes,
 264                         unsigned long long const &FileSize)
 265 {
 266    // See if this isn't an internaly prefix'd file name.
 267    if (InternalPrefix.empty() == false &&
 268        InternalPrefix.length() < FileName.length() &&
 269        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 270                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 271    {
 272       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 273       {
 274          // Tidy up the display
 275          if (DeLinkBytes == 0)
 276             cout << endl;
 277
 278          NewLine(1);
 279          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 280                     SizeToStr(FileSize).c_str());
 281          c1out << flush;
 282
 283          if (NoLinkAct == false)
 284          {
 285             char OldLink[400];
 286             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 287                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 288             else
 289             {
 290                if (unlink(OriginalPath) != 0)
 291                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 292                else
 293                {
 294                   if (link(FileName.c_str(),OriginalPath) != 0)
 295                   {
 296                      // Panic! Restore the symlink
 297                      if (symlink(OldLink,OriginalPath) != 0)
 298                         _error->Errno("symlink", "failed to restore symlink");
 299                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 300                                           FileName.c_str(),
 301                                           OriginalPath);
 302                   }
 303                }
 304             }
 305          }
 306
 307          DeLinkBytes += FileSize;
 308          if (DeLinkBytes/1024 >= DeLinkLimit)
 309             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 310       }
 311
 312       FileName = OriginalPath;
 313    }
 314
 315    return true;
 316 }
 317                                                                         /*}}}*/
 318
 319 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 320 // ---------------------------------------------------------------------
 321 /* */
 322 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 323                                string const &Arch) :
 324    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 325 {
 326    Output = stdout;
 327    SetExts(".deb .udeb");
 328    DeLinkLimit = 0;
 329
 330    // Process the command line options
 331    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 332    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 333    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 334    DoSHA512 = _config->FindB("APT::FTPArchive::Packages::SHA512",DoSHA512);
 335    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 336    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 337    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 338    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 339
 340    if (Db.Loaded() == false)
 341       DoContents = false;
 342
 343    // Read the override file
 344    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 345       return;
 346    else
 347       NoOverride = true;
 348
 349    if (ExtOverrides.empty() == false)
 350       Over.ReadExtraOverride(ExtOverrides);
 351
 352    _error->DumpErrors();
 353 }
 354                                                                         /*}}}*/
 355 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 356 // ---------------------------------------------------------------------
 357 /* */
 358 bool FTWScanner::SetExts(string const &Vals)
 359 {
 360    ClearPatterns();
 361    string::size_type Start = 0;
 362    while (Start <= Vals.length()-1)
 363    {
 364       string::size_type const Space = Vals.find(' ',Start);
 365       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 366       if ( Arch.empty() == false )
 367       {
 368          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 369          AddPattern(string("*_all") + Vals.substr(Start, Length));
 370       }
 371       else
 372          AddPattern(string("*") + Vals.substr(Start, Length));
 373
 374       Start += Length + 1;
 375    }
 376
 377    return true;
 378 }
 379
 380                                                                         /*}}}*/
 381 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 382 // ---------------------------------------------------------------------
 383 /* This method takes a package and gets its control information and
 384    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 385    rewritten and the path/size/hash appended. */
 386 bool PackagesWriter::DoPackage(string FileName)
 387 {
 388    // Pull all the data we need form the DB
 389    if (Db.GetFileInfo(FileName,
 390                       true,   /* DoControl */
 391                       DoContents,
 392                       true,   /* GenContentsOnly */
 393                       false,  /* DoSource */
 394                       DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false)
 395    {
 396      return false;
 397    }
 398
 399    unsigned long long FileSize = Db.GetFileSize();
 400    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 401       return false;
 402
 403    // Lookup the overide information
 404    pkgTagSection &Tags = Db.Control.Section;
 405    string Package = Tags.FindS("Package");
 406    string Architecture;
 407    // if we generate a Packages file for a given arch, we use it to
 408    // look for overrides. if we run in "simple" mode without the
 409    // "Architecures" variable in the config we use the architecure value
 410    // from the deb file
 411    if(Arch != "")
 412       Architecture = Arch;
 413    else
 414       Architecture = Tags.FindS("Architecture");
 415    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 416
 417    if (Package.empty() == true)
 418       return _error->Error(_("Archive had no package field"));
 419
 420    // If we need to do any rewriting of the header do it now..
 421    if (OverItem.get() == 0)
 422    {
 423       if (NoOverride == false)
 424       {
 425          NewLine(1);
 426          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 427       }
 428
 429       OverItem = auto_ptr<Override::Item>(new Override::Item);
 430       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 431       OverItem->Priority = Tags.FindS("Priority");
 432    }
 433
 434    char Size[40];
 435    sprintf(Size,"%llu", (unsigned long long) FileSize);
 436
 437    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 438    string NewFileName;
 439    if (DirStrip.empty() == false &&
 440        FileName.length() > DirStrip.length() &&
 441        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 442                  DirStrip.begin(),DirStrip.end()) == 0)
 443       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 444    else
 445       NewFileName = FileName;
 446    if (PathPrefix.empty() == false)
 447       NewFileName = flCombine(PathPrefix,NewFileName);
 448
 449    /* Configuration says we don't want to include the long Description
 450       in the package file - instead we want to ship a separated file */
 451    string desc;
 452    if (LongDescription == false) {
 453       desc = Tags.FindS("Description").append("\n");
 454       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 455    }
 456
 457    // This lists all the changes to the fields we are going to make.
 458    std::vector<TFRewriteData> Changes;
 459
 460    Changes.push_back(SetTFRewriteData("Size", Size));
 461    if (DoMD5 == true)
 462       Changes.push_back(SetTFRewriteData("MD5sum", Db.MD5Res.c_str()));
 463    if (DoSHA1 == true)
 464       Changes.push_back(SetTFRewriteData("SHA1", Db.SHA1Res.c_str()));
 465    if (DoSHA256 == true)
 466       Changes.push_back(SetTFRewriteData("SHA256", Db.SHA256Res.c_str()));
 467    if (DoSHA512 == true)
 468       Changes.push_back(SetTFRewriteData("SHA512", Db.SHA512Res.c_str()));
 469    Changes.push_back(SetTFRewriteData("Filename", NewFileName.c_str()));
 470    Changes.push_back(SetTFRewriteData("Priority", OverItem->Priority.c_str()));
 471    Changes.push_back(SetTFRewriteData("Status", 0));
 472    Changes.push_back(SetTFRewriteData("Optional", 0));
 473
 474    string DescriptionMd5;
 475    if (LongDescription == false) {
 476       MD5Summation descmd5;
 477       descmd5.Add(desc.c_str());
 478       DescriptionMd5 = descmd5.Result().Value();
 479       Changes.push_back(SetTFRewriteData("Description-md5", DescriptionMd5.c_str()));
 480       if (TransWriter != NULL)
 481          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 482    }
 483
 484    // Rewrite the maintainer field if necessary
 485    bool MaintFailed;
 486    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 487    if (MaintFailed == true)
 488    {
 489       if (NoOverride == false)
 490       {
 491          NewLine(1);
 492          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 493                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 494       }
 495    }
 496
 497    if (NewMaint.empty() == false)
 498       Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
 499
 500    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 501       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 502       but dpkg does this append bit. So we do the append bit, at least that way the
 503       status file and package file will remain similar. There are other transforms
 504       but optional is the only legacy one still in use for some lazy reason. */
 505    string OptionalStr = Tags.FindS("Optional");
 506    if (OptionalStr.empty() == false)
 507    {
 508       if (Tags.FindS("Suggests").empty() == false)
 509          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 510       Changes.push_back(SetTFRewriteData("Suggests", OptionalStr.c_str()));
 511    }
 512
 513    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 514         I != OverItem->FieldOverride.end(); ++I)
 515       Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
 516
 517    Changes.push_back(SetTFRewriteData( 0, 0));
 518
 519    // Rewrite and store the fields.
 520    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes.data()) == false)
 521       return false;
 522    fprintf(Output,"\n");
 523
 524    return Db.Finish();
 525 }
 526                                                                         /*}}}*/
 527
 528 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 529 // ---------------------------------------------------------------------
 530 /* Create a Translation-Master file for this Packages file */
 531 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 532                                         mode_t const &Permissions) : Output(NULL),
 533                                                         RefCounter(0)
 534 {
 535    if (File.empty() == true)
 536       return;
 537
 538    Comp = new MultiCompress(File, TransCompress, Permissions);
 539    Output = Comp->Input;
 540 }
 541                                                                         /*}}}*/
 542 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 543 // ---------------------------------------------------------------------
 544 /* Create a Translation-Master file for this Packages file */
 545 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 546                                   string const &MD5)
 547 {
 548    if (Output == NULL)
 549       return true;
 550
 551    // Different archs can include different versions and therefore
 552    // different descriptions - so we need to check for both name and md5.
 553    string const Record = Pkg + ":" + MD5;
 554
 555    if (Included.find(Record) != Included.end())
 556       return true;
 557
 558    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 559            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 560
 561    Included.insert(Record);
 562    return true;
 563 }
 564                                                                         /*}}}*/
 565 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 566 // ---------------------------------------------------------------------
 567 /* */
 568 TranslationWriter::~TranslationWriter()
 569 {
 570    if (Comp == NULL)
 571       return;
 572
 573    delete Comp;
 574 }
 575                                                                         /*}}}*/
 576
 577 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 578 // ---------------------------------------------------------------------
 579 /* */
 580 SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
 581                              string const &ExtOverrides) :
 582    Db(DB), Stats(Db.Stats)
 583 {
 584    Output = stdout;
 585    AddPattern("*.dsc");
 586    DeLinkLimit = 0;
 587    Buffer = 0;
 588    BufSize = 0;
 589
 590    // Process the command line options
 591    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 592    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 593    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 594    DoSHA512 = _config->FindB("APT::FTPArchive::Sources::SHA512",DoSHA512);
 595    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 596    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 597
 598    // Read the override file
 599    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 600       return;
 601    else
 602       NoOverride = true;
 603
 604    // WTF?? The logic above: if we can't read binary overrides, don't even try
 605    // reading source overrides. if we can read binary overrides, then say there
 606    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 607
 608    if (ExtOverrides.empty() == false)
 609       SOver.ReadExtraOverride(ExtOverrides);
 610
 611    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 612       SOver.ReadOverride(SOverrides,true);
 613 }
 614                                                                         /*}}}*/
 615 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 616 // ---------------------------------------------------------------------
 617 /* */
 618 bool SourcesWriter::DoPackage(string FileName)
 619 {
 620    // Pull all the data we need form the DB
 621    if (Db.GetFileInfo(FileName,
 622                       false,  /* DoControl */
 623                       false,  /* DoContents */
 624                       false,  /* GenContentsOnly */
 625                       true,   /* DoSource */
 626                       DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false)
 627    {
 628       return false;
 629    }
 630
 631    // we need to perform a "write" here (this is what finish is doing)
 632    // because the call to Db.GetFileInfo() in the loop will change
 633    // the "db cursor"
 634    Db.Finish();
 635
 636    // read stuff
 637    char *Start = Db.Dsc.Data;
 638    char *BlkEnd = Db.Dsc.Data + Db.Dsc.Length;
 639
 640    // Add extra \n to the end, just in case (as in clearsigned they are missing)
 641    *BlkEnd++ = '\n';
 642    *BlkEnd++ = '\n';
 643
 644    pkgTagSection Tags;
 645    if (Tags.Scan(Start,BlkEnd - Start) == false)
 646       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 647
 648    if (Tags.Exists("Source") == false)
 649       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
 650    Tags.Trim();
 651
 652    // Lookup the overide information, finding first the best priority.
 653    string BestPrio;
 654    string Bins = Tags.FindS("Binary");
 655    char Buffer[Bins.length() + 1];
 656    auto_ptr<Override::Item> OverItem(0);
 657    if (Bins.empty() == false)
 658    {
 659       strcpy(Buffer,Bins.c_str());
 660
 661       // Ignore too-long errors.
 662       char *BinList[400];
 663       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 664
 665       // Look at all the binaries
 666       unsigned char BestPrioV = pkgCache::State::Extra;
 667       for (unsigned I = 0; BinList[I] != 0; I++)
 668       {
 669          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 670          if (Itm.get() == 0)
 671             continue;
 672
 673          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 674          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 675          {
 676             BestPrioV = NewPrioV;
 677             BestPrio = Itm->Priority;
 678          }
 679
 680          if (OverItem.get() == 0)
 681             OverItem = Itm;
 682       }
 683    }
 684
 685    // If we need to do any rewriting of the header do it now..
 686    if (OverItem.get() == 0)
 687    {
 688       if (NoOverride == false)
 689       {
 690          NewLine(1);
 691          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 692       }
 693
 694       OverItem = auto_ptr<Override::Item>(new Override::Item);
 695    }
 696
 697    struct stat St;
 698    if (stat(FileName.c_str(), &St) != 0)
 699       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 700
 701    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 702    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 703    if (SOverItem.get() == 0)
 704    {
 705       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 706       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 707       if (SOverItem.get() == 0)
 708       {
 709         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 710          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 711          *SOverItem = *OverItem;
 712       }
 713    }
 714
 715    // Add the dsc to the files hash list
 716    string const strippedName = flNotDir(FileName);
 717    std::ostringstream ostreamFiles;
 718    if (DoMD5 == true && Tags.Exists("Files"))
 719       ostreamFiles << "\n " << Db.MD5Res.c_str() << " " << St.st_size << " "
 720                    << strippedName << "\n " << Tags.FindS("Files");
 721    string const Files = ostreamFiles.str();
 722
 723    std::ostringstream ostreamSha1;
 724    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 725       ostreamSha1 << "\n " << string(Db.SHA1Res.c_str()) << " " << St.st_size << " "
 726                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 727
 728    std::ostringstream ostreamSha256;
 729    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 730       ostreamSha256 << "\n " << string(Db.SHA256Res.c_str()) << " " << St.st_size << " "
 731                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 732
 733    std::ostringstream ostreamSha512;
 734    if (DoSHA512 == true && Tags.Exists("Checksums-Sha512"))
 735       ostreamSha512 << "\n " << string(Db.SHA512Res.c_str()) << " " << St.st_size << " "
 736                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 737
 738    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 739    string NewFileName;
 740    if (DirStrip.empty() == false &&
 741        FileName.length() > DirStrip.length() &&
 742        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 743       NewFileName = string(OriginalPath + DirStrip.length());
 744    else
 745       NewFileName = OriginalPath;
 746    if (PathPrefix.empty() == false)
 747       NewFileName = flCombine(PathPrefix,NewFileName);
 748
 749    string Directory = flNotFile(OriginalPath);
 750    string Package = Tags.FindS("Source");
 751
 752    // Perform operation over all of the files
 753    string ParseJnk;
 754    const char *C = Files.c_str();
 755    char *RealPath = NULL;
 756    for (;isspace(*C); C++);
 757    while (*C != 0)
 758    {
 759       // Parse each of the elements
 760       if (ParseQuoteWord(C,ParseJnk) == false ||
 761           ParseQuoteWord(C,ParseJnk) == false ||
 762           ParseQuoteWord(C,ParseJnk) == false)
 763          return _error->Error("Error parsing file record");
 764
 765       string OriginalPath = Directory + ParseJnk;
 766
 767       // Add missing hashes to source files
 768       if ((DoSHA1 == true && !Tags.Exists("Checksums-Sha1")) ||
 769           (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) ||
 770           (DoSHA512 == true && !Tags.Exists("Checksums-Sha512")))
 771       {
 772          if (Db.GetFileInfo(OriginalPath,
 773                             false, /* DoControl */
 774                             false, /* DoContents */
 775                             false, /* GenContentsOnly */
 776                             false, /* DoSource */
 777                             DoMD5, DoSHA1, DoSHA256, DoSHA512,
 778                             DoAlwaysStat) == false)
 779          {
 780             return _error->Error("Error getting file info");
 781          }
 782
 783          if (DoSHA1 == true && !Tags.Exists("Checksums-Sha1"))
 784             ostreamSha1 << "\n " << string(Db.SHA1Res) << " "
 785                << Db.GetFileSize() << " " << ParseJnk;
 786
 787          if (DoSHA256 == true && !Tags.Exists("Checksums-Sha256"))
 788             ostreamSha256 << "\n " << string(Db.SHA256Res) << " "
 789                << Db.GetFileSize() << " " << ParseJnk;
 790
 791          if (DoSHA512 == true && !Tags.Exists("Checksums-Sha512"))
 792             ostreamSha512 << "\n " << string(Db.SHA512Res) << " "
 793                << Db.GetFileSize() << " " << ParseJnk;
 794
 795          // write back the GetFileInfo() stats data
 796          Db.Finish();
 797       }
 798
 799       // Perform the delinking operation
 800       char Jnk[2];
 801
 802       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 803           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 804       {
 805          string RP = RealPath;
 806          free(RealPath);
 807          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 808             return false;
 809       }
 810    }
 811
 812    Directory = flNotFile(NewFileName);
 813    if (Directory.length() > 2)
 814       Directory.erase(Directory.end()-1);
 815
 816    string const ChecksumsSha1 = ostreamSha1.str();
 817    string const ChecksumsSha256 = ostreamSha256.str();
 818    string const ChecksumsSha512 = ostreamSha512.str();
 819
 820    // This lists all the changes to the fields we are going to make.
 821    // (5 hardcoded + checksums + maintainer + end marker)
 822    std::vector<TFRewriteData> Changes;
 823
 824    Changes.push_back(SetTFRewriteData("Source",Package.c_str(),"Package"));
 825    if (Files.empty() == false)
 826       Changes.push_back(SetTFRewriteData("Files",Files.c_str()));
 827    if (ChecksumsSha1.empty() == false)
 828       Changes.push_back(SetTFRewriteData("Checksums-Sha1",ChecksumsSha1.c_str()));
 829    if (ChecksumsSha256.empty() == false)
 830       Changes.push_back(SetTFRewriteData("Checksums-Sha256",ChecksumsSha256.c_str()));
 831    if (ChecksumsSha512.empty() == false)
 832       Changes.push_back(SetTFRewriteData("Checksums-Sha512",ChecksumsSha512.c_str()));
 833    if (Directory != "./")
 834       Changes.push_back(SetTFRewriteData("Directory",Directory.c_str()));
 835    Changes.push_back(SetTFRewriteData("Priority",BestPrio.c_str()));
 836    Changes.push_back(SetTFRewriteData("Status",0));
 837
 838    // Rewrite the maintainer field if necessary
 839    bool MaintFailed;
 840    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 841    if (MaintFailed == true)
 842    {
 843       if (NoOverride == false)
 844       {
 845          NewLine(1);
 846          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 847                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 848       }
 849    }
 850    if (NewMaint.empty() == false)
 851       Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
 852
 853    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 854         I != SOverItem->FieldOverride.end(); ++I)
 855       Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
 856
 857    Changes.push_back(SetTFRewriteData(0, 0));
 858
 859    // Rewrite and store the fields.
 860    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes.data()) == false)
 861       return false;
 862    fprintf(Output,"\n");
 863
 864    Stats.Packages++;
 865
 866    return true;
 867 }
 868                                                                         /*}}}*/
 869
 870 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 871 // ---------------------------------------------------------------------
 872 /* */
 873 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 874                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 875
 876 {
 877    SetExts(".deb");
 878    Output = stdout;
 879 }
 880                                                                         /*}}}*/
 881 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 882 // ---------------------------------------------------------------------
 883 /* If Package is the empty string the control record will be parsed to
 884    determine what the package name is. */
 885 bool ContentsWriter::DoPackage(string FileName, string Package)
 886 {
 887    if (!Db.GetFileInfo(FileName,
 888                        Package.empty(), /* DoControl */
 889                        true,            /* DoContents */
 890                        false,           /* GenContentsOnly */
 891                        false,           /* DoSource */
 892                        false,           /* DoMD5 */
 893                        false,           /* DoSHA1 */
 894                        false,           /* DoSHA256 */
 895                        false))          /* DoSHA512 */
 896    {
 897       return false;
 898    }
 899
 900    // Parse the package name
 901    if (Package.empty() == true)
 902    {
 903       Package = Db.Control.Section.FindS("Package");
 904    }
 905
 906    Db.Contents.Add(Gen,Package);
 907
 908    return Db.Finish();
 909 }
 910                                                                         /*}}}*/
 911 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 912 // ---------------------------------------------------------------------
 913 /* */
 914 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 915 {
 916    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 917    if (_error->PendingError() == true)
 918       return false;
 919
 920    // Open the package file
 921    FileFd Fd;
 922    if (Pkgs.OpenOld(Fd) == false)
 923       return false;
 924
 925    pkgTagFile Tags(&Fd);
 926    if (_error->PendingError() == true)
 927       return false;
 928
 929    // Parse.
 930    pkgTagSection Section;
 931    while (Tags.Step(Section) == true)
 932    {
 933       string File = flCombine(Prefix,Section.FindS("FileName"));
 934       string Package = Section.FindS("Section");
 935       if (Package.empty() == false && Package.end()[-1] != '/')
 936       {
 937          Package += '/';
 938          Package += Section.FindS("Package");
 939       }
 940       else
 941          Package += Section.FindS("Package");
 942
 943       DoPackage(File,Package);
 944       if (_error->empty() == false)
 945       {
 946          _error->Error("Errors apply to file '%s'",File.c_str());
 947          _error->DumpErrors();
 948       }
 949    }
 950
 951    // Tidy the compressor
 952    Fd.Close();
 953
 954    return true;
 955 }
 956
 957                                                                         /*}}}*/
 958
 959 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 960 // ---------------------------------------------------------------------
 961 /* */
 962 ReleaseWriter::ReleaseWriter(string const &/*DB*/)
 963 {
 964    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 965    {
 966       AddPattern("Packages");
 967       AddPattern("Packages.gz");
 968       AddPattern("Packages.bz2");
 969       AddPattern("Packages.lzma");
 970       AddPattern("Packages.xz");
 971       AddPattern("Translation-*");
 972       AddPattern("Sources");
 973       AddPattern("Sources.gz");
 974       AddPattern("Sources.bz2");
 975       AddPattern("Sources.lzma");
 976       AddPattern("Sources.xz");
 977       AddPattern("Release");
 978       AddPattern("Contents-*");
 979       AddPattern("Index");
 980       AddPattern("md5sum.txt");
 981    }
 982    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 983
 984    Output = stdout;
 985    time_t const now = time(NULL);
 986
 987    setlocale(LC_TIME, "C");
 988
 989    char datestr[128];
 990    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 991                 gmtime(&now)) == 0)
 992    {
 993       datestr[0] = '\0';
 994    }
 995
 996    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 997    char validstr[128];
 998    if (now == validuntil ||
 999        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
1000                 gmtime(&validuntil)) == 0)
1001    {
1002       validstr[0] = '\0';
1003    }
1004
1005    setlocale(LC_TIME, "");
1006
1007    map<string,string> Fields;
1008    Fields["Origin"] = "";
1009    Fields["Label"] = "";
1010    Fields["Suite"] = "";
1011    Fields["Version"] = "";
1012    Fields["Codename"] = "";
1013    Fields["Date"] = datestr;
1014    Fields["Valid-Until"] = validstr;
1015    Fields["Architectures"] = "";
1016    Fields["Components"] = "";
1017    Fields["Description"] = "";
1018
1019    for(map<string,string>::const_iterator I = Fields.begin();
1020        I != Fields.end();
1021        ++I)
1022    {
1023       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1024       string Value = _config->Find(Config, (*I).second.c_str());
1025       if (Value == "")
1026          continue;
1027
1028       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1029    }
1030
1031    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1032    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1033    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1034 }
1035                                                                         /*}}}*/
1036 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1037 // ---------------------------------------------------------------------
1038 bool ReleaseWriter::DoPackage(string FileName)
1039 {
1040    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1041    string NewFileName;
1042    if (DirStrip.empty() == false &&
1043        FileName.length() > DirStrip.length() &&
1044        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1045                  DirStrip.begin(),DirStrip.end()) == 0)
1046    {
1047       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1048       while (NewFileName[0] == '/')
1049          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1050    }
1051    else
1052       NewFileName = FileName;
1053
1054    if (PathPrefix.empty() == false)
1055       NewFileName = flCombine(PathPrefix,NewFileName);
1056
1057    FileFd fd(FileName, FileFd::ReadOnly);
1058
1059    if (!fd.IsOpen())
1060    {
1061       return false;
1062    }
1063
1064    CheckSums[NewFileName].size = fd.Size();
1065
1066    Hashes hs;
1067    hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1068    if (DoMD5 == true)
1069       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1070    if (DoSHA1 == true)
1071       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1072    if (DoSHA256 == true)
1073       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1074    if (DoSHA512 == true)
1075       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1076    fd.Close();
1077
1078    return true;
1079 }
1080
1081                                                                         /*}}}*/
1082 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1083 // ---------------------------------------------------------------------
1084 void ReleaseWriter::Finish()
1085 {
1086    if (DoMD5 == true)
1087    {
1088       fprintf(Output, "MD5Sum:\n");
1089       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1090           I != CheckSums.end(); ++I)
1091       {
1092          fprintf(Output, " %s %16llu %s\n",
1093                  (*I).second.MD5.c_str(),
1094                  (*I).second.size,
1095                  (*I).first.c_str());
1096       }
1097    }
1098    if (DoSHA1 == true)
1099    {
1100       fprintf(Output, "SHA1:\n");
1101       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1102           I != CheckSums.end(); ++I)
1103       {
1104          fprintf(Output, " %s %16llu %s\n",
1105                  (*I).second.SHA1.c_str(),
1106                  (*I).second.size,
1107                  (*I).first.c_str());
1108       }
1109    }
1110    if (DoSHA256 == true)
1111    {
1112       fprintf(Output, "SHA256:\n");
1113       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1114           I != CheckSums.end(); ++I)
1115       {
1116          fprintf(Output, " %s %16llu %s\n",
1117                  (*I).second.SHA256.c_str(),
1118                  (*I).second.size,
1119                  (*I).first.c_str());
1120       }
1121    }
1122
1123    fprintf(Output, "SHA512:\n");
1124    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1125        I != CheckSums.end();
1126        ++I)
1127    {
1128       fprintf(Output, " %s %16llu %s\n",
1129               (*I).second.SHA512.c_str(),
1130               (*I).second.size,
1131               (*I).first.c_str());
1132    }
1133
1134 }