ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/deblistparser.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/fileutl.h>
  20 #include <apt-pkg/gpgv.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/debfile.h>
  25 #include <apt-pkg/pkgcache.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/sha2.h>
  28 #include <apt-pkg/tagfile.h>
  29
  30 #include <ctype.h>
  31 #include <fnmatch.h>
  32 #include <ftw.h>
  33 #include <locale.h>
  34 #include <string.h>
  35 #include <sys/stat.h>
  36 #include <sys/types.h>
  37 #include <unistd.h>
  38 #include <ctime>
  39 #include <iostream>
  40 #include <sstream>
  41 #include <memory>
  42 #include <utility>
  43
  44 #include "apt-ftparchive.h"
  45 #include "writer.h"
  46 #include "cachedb.h"
  47 #include "multicompress.h"
  48
  49 #include <apti18n.h>
  50                                                                         /*}}}*/
  51 using namespace std;
  52 FTWScanner *FTWScanner::Owner;
  53
  54 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  58                              const char *tag,
  59                              const char *rewrite,
  60                              const char *newtag = 0)
  61 {
  62     tfrd.Tag = tag;
  63     tfrd.Rewrite = rewrite;
  64     tfrd.NewTag = newtag;
  65 }
  66                                                                         /*}}}*/
  67
  68 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* */
  71 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  72 {
  73    ErrorPrinted = false;
  74    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  75
  76    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  77    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  78    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  79    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  80 }
  81                                                                         /*}}}*/
  82 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  83 // ---------------------------------------------------------------------
  84 /* This is the FTW scanner, it processes each directory element in the
  85    directory tree. */
  86 int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
  87 {
  88    if (Flag == FTW_DNR)
  89    {
  90       Owner->NewLine(1);
  91       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  92    }
  93    if (Flag == FTW_NS)
  94    {
  95       Owner->NewLine(1);
  96       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  97    }
  98    if (Flag != FTW_F)
  99       return 0;
 100
 101    return ScannerFile(File, true);
 102 }
 103                                                                         /*}}}*/
 104 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
 105 // ---------------------------------------------------------------------
 106 /* */
 107 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
 108 {
 109    const char *LastComponent = strrchr(File, '/');
 110    char *RealPath = NULL;
 111
 112    if (LastComponent == NULL)
 113       LastComponent = File;
 114    else
 115       LastComponent++;
 116
 117    vector<string>::const_iterator I;
 118    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 119    {
 120       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 121          break;
 122    }
 123    if (I == Owner->Patterns.end())
 124       return 0;
 125
 126    /* Process it. If the file is a link then resolve it into an absolute
 127       name.. This works best if the directory components the scanner are
 128       given are not links themselves. */
 129    char Jnk[2];
 130    Owner->OriginalPath = File;
 131    if (ReadLink &&
 132        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 133        (RealPath = realpath(File,NULL)) != 0)
 134    {
 135       Owner->DoPackage(RealPath);
 136       free(RealPath);
 137    }
 138    else
 139       Owner->DoPackage(File);
 140
 141    if (_error->empty() == false)
 142    {
 143       // Print any errors or warnings found
 144       string Err;
 145       bool SeenPath = false;
 146       while (_error->empty() == false)
 147       {
 148          Owner->NewLine(1);
 149
 150          bool const Type = _error->PopMessage(Err);
 151          if (Type == true)
 152             cerr << _("E: ") << Err << endl;
 153          else
 154             cerr << _("W: ") << Err << endl;
 155
 156          if (Err.find(File) != string::npos)
 157             SeenPath = true;
 158       }
 159
 160       if (SeenPath == false)
 161          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 162       return 0;
 163    }
 164
 165    return 0;
 166 }
 167                                                                         /*}}}*/
 168 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 169 // ---------------------------------------------------------------------
 170 /* */
 171 bool FTWScanner::RecursiveScan(string const &Dir)
 172 {
 173    char *RealPath = NULL;
 174    /* If noprefix is set then jam the scan root in, so we don't generate
 175       link followed paths out of control */
 176    if (InternalPrefix.empty() == true)
 177    {
 178       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 179          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 180       InternalPrefix = RealPath;
 181       free(RealPath);
 182    }
 183
 184    // Do recursive directory searching
 185    Owner = this;
 186    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 187
 188    // Error treewalking?
 189    if (Res != 0)
 190    {
 191       if (_error->PendingError() == false)
 192          _error->Errno("ftw",_("Tree walking failed"));
 193       return false;
 194    }
 195
 196    return true;
 197 }
 198                                                                         /*}}}*/
 199 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 200 // ---------------------------------------------------------------------
 201 /* This is an alternative to using FTW to locate files, it reads the list
 202    of files from another file. */
 203 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 204 {
 205    char *RealPath = NULL;
 206    /* If noprefix is set then jam the scan root in, so we don't generate
 207       link followed paths out of control */
 208    if (InternalPrefix.empty() == true)
 209    {
 210       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 211          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 212       InternalPrefix = RealPath;
 213       free(RealPath);
 214    }
 215
 216    Owner = this;
 217    FILE *List = fopen(File.c_str(),"r");
 218    if (List == 0)
 219       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 220
 221    /* We are a tad tricky here.. We prefix the buffer with the directory
 222       name, that way if we need a full path with just use line.. Sneaky and
 223       fully evil. */
 224    char Line[1000];
 225    char *FileStart;
 226    if (Dir.empty() == true || Dir.end()[-1] != '/')
 227       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 228    else
 229       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 230    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 231    {
 232       char *FileName = _strstrip(FileStart);
 233       if (FileName[0] == 0)
 234          continue;
 235
 236       if (FileName[0] != '/')
 237       {
 238          if (FileName != FileStart)
 239             memmove(FileStart,FileName,strlen(FileStart));
 240          FileName = Line;
 241       }
 242
 243 #if 0
 244       struct stat St;
 245       int Flag = FTW_F;
 246       if (stat(FileName,&St) != 0)
 247          Flag = FTW_NS;
 248 #endif
 249
 250       if (ScannerFile(FileName, false) != 0)
 251          break;
 252    }
 253
 254    fclose(List);
 255    return true;
 256 }
 257                                                                         /*}}}*/
 258 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* */
 261 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 262                         unsigned long long &DeLinkBytes,
 263                         unsigned long long const &FileSize)
 264 {
 265    // See if this isn't an internaly prefix'd file name.
 266    if (InternalPrefix.empty() == false &&
 267        InternalPrefix.length() < FileName.length() &&
 268        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 269                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 270    {
 271       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 272       {
 273          // Tidy up the display
 274          if (DeLinkBytes == 0)
 275             cout << endl;
 276
 277          NewLine(1);
 278          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 279                     SizeToStr(FileSize).c_str());
 280          c1out << flush;
 281
 282          if (NoLinkAct == false)
 283          {
 284             char OldLink[400];
 285             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 286                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 287             else
 288             {
 289                if (unlink(OriginalPath) != 0)
 290                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 291                else
 292                {
 293                   if (link(FileName.c_str(),OriginalPath) != 0)
 294                   {
 295                      // Panic! Restore the symlink
 296                      if (symlink(OldLink,OriginalPath) != 0)
 297                         _error->Errno("symlink", "failed to restore symlink");
 298                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 299                                           FileName.c_str(),
 300                                           OriginalPath);
 301                   }
 302                }
 303             }
 304          }
 305
 306          DeLinkBytes += FileSize;
 307          if (DeLinkBytes/1024 >= DeLinkLimit)
 308             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 309       }
 310
 311       FileName = OriginalPath;
 312    }
 313
 314    return true;
 315 }
 316                                                                         /*}}}*/
 317
 318 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 319 // ---------------------------------------------------------------------
 320 /* */
 321 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 322                                string const &Arch) :
 323    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 324 {
 325    Output = stdout;
 326    SetExts(".deb .udeb");
 327    DeLinkLimit = 0;
 328
 329    // Process the command line options
 330    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 331    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 332    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 333    DoSHA512 = _config->FindB("APT::FTPArchive::Packages::SHA512",DoSHA512);
 334    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 335    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 336    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 337    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 338
 339    if (Db.Loaded() == false)
 340       DoContents = false;
 341
 342    // Read the override file
 343    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 344       return;
 345    else
 346       NoOverride = true;
 347
 348    if (ExtOverrides.empty() == false)
 349       Over.ReadExtraOverride(ExtOverrides);
 350
 351    _error->DumpErrors();
 352 }
 353                                                                         /*}}}*/
 354 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* */
 357 bool FTWScanner::SetExts(string const &Vals)
 358 {
 359    ClearPatterns();
 360    string::size_type Start = 0;
 361    while (Start <= Vals.length()-1)
 362    {
 363       string::size_type const Space = Vals.find(' ',Start);
 364       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 365       if ( Arch.empty() == false )
 366       {
 367          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 368          AddPattern(string("*_all") + Vals.substr(Start, Length));
 369       }
 370       else
 371          AddPattern(string("*") + Vals.substr(Start, Length));
 372
 373       Start += Length + 1;
 374    }
 375
 376    return true;
 377 }
 378
 379                                                                         /*}}}*/
 380 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* This method takes a package and gets its control information and
 383    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 384    rewritten and the path/size/hash appended. */
 385 bool PackagesWriter::DoPackage(string FileName)
 386 {
 387    // Pull all the data we need form the DB
 388    if (Db.GetFileInfo(FileName,
 389                       true,   /* DoControl */
 390                       DoContents,
 391                       true,   /* GenContentsOnly */
 392                       false,  /* DoSource */
 393                       DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false)
 394    {
 395      return false;
 396    }
 397
 398    unsigned long long FileSize = Db.GetFileSize();
 399    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 400       return false;
 401
 402    // Lookup the overide information
 403    pkgTagSection &Tags = Db.Control.Section;
 404    string Package = Tags.FindS("Package");
 405    string Architecture;
 406    // if we generate a Packages file for a given arch, we use it to
 407    // look for overrides. if we run in "simple" mode without the
 408    // "Architecures" variable in the config we use the architecure value
 409    // from the deb file
 410    if(Arch != "")
 411       Architecture = Arch;
 412    else
 413       Architecture = Tags.FindS("Architecture");
 414    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 415
 416    if (Package.empty() == true)
 417       return _error->Error(_("Archive had no package field"));
 418
 419    // If we need to do any rewriting of the header do it now..
 420    if (OverItem.get() == 0)
 421    {
 422       if (NoOverride == false)
 423       {
 424          NewLine(1);
 425          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 426       }
 427
 428       OverItem = auto_ptr<Override::Item>(new Override::Item);
 429       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 430       OverItem->Priority = Tags.FindS("Priority");
 431    }
 432
 433    char Size[40];
 434    sprintf(Size,"%llu", (unsigned long long) FileSize);
 435
 436    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 437    string NewFileName;
 438    if (DirStrip.empty() == false &&
 439        FileName.length() > DirStrip.length() &&
 440        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 441                  DirStrip.begin(),DirStrip.end()) == 0)
 442       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 443    else
 444       NewFileName = FileName;
 445    if (PathPrefix.empty() == false)
 446       NewFileName = flCombine(PathPrefix,NewFileName);
 447
 448    /* Configuration says we don't want to include the long Description
 449       in the package file - instead we want to ship a separated file */
 450    string desc;
 451    if (LongDescription == false) {
 452       desc = Tags.FindS("Description").append("\n");
 453       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 454    }
 455
 456    // This lists all the changes to the fields we are going to make.
 457    // (7 hardcoded + maintainer + suggests + end marker)
 458    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 459
 460    unsigned int End = 0;
 461    SetTFRewriteData(Changes[End++], "Size", Size);
 462    if (DoMD5 == true)
 463       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 464    if (DoSHA1 == true)
 465       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 466    if (DoSHA256 == true)
 467       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 468    if (DoSHA512 == true)
 469       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 470    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 471    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 472    SetTFRewriteData(Changes[End++], "Status", 0);
 473    SetTFRewriteData(Changes[End++], "Optional", 0);
 474
 475    string DescriptionMd5;
 476    if (LongDescription == false) {
 477       MD5Summation descmd5;
 478       descmd5.Add(desc.c_str());
 479       DescriptionMd5 = descmd5.Result().Value();
 480       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 481       if (TransWriter != NULL)
 482          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 483    }
 484
 485    // Rewrite the maintainer field if necessary
 486    bool MaintFailed;
 487    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 488    if (MaintFailed == true)
 489    {
 490       if (NoOverride == false)
 491       {
 492          NewLine(1);
 493          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 494                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 495       }
 496    }
 497
 498    if (NewMaint.empty() == false)
 499       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 500
 501    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 502       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 503       but dpkg does this append bit. So we do the append bit, at least that way the
 504       status file and package file will remain similar. There are other transforms
 505       but optional is the only legacy one still in use for some lazy reason. */
 506    string OptionalStr = Tags.FindS("Optional");
 507    if (OptionalStr.empty() == false)
 508    {
 509       if (Tags.FindS("Suggests").empty() == false)
 510          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 511       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 512    }
 513
 514    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 515         I != OverItem->FieldOverride.end(); ++I)
 516       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 517
 518    SetTFRewriteData(Changes[End++], 0, 0);
 519
 520    // Rewrite and store the fields.
 521    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 522       return false;
 523    fprintf(Output,"\n");
 524
 525    return Db.Finish();
 526 }
 527                                                                         /*}}}*/
 528
 529 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* Create a Translation-Master file for this Packages file */
 532 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 533                                         mode_t const &Permissions) : Output(NULL),
 534                                                         RefCounter(0)
 535 {
 536    if (File.empty() == true)
 537       return;
 538
 539    Comp = new MultiCompress(File, TransCompress, Permissions);
 540    Output = Comp->Input;
 541 }
 542                                                                         /*}}}*/
 543 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* Create a Translation-Master file for this Packages file */
 546 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 547                                   string const &MD5)
 548 {
 549    if (Output == NULL)
 550       return true;
 551
 552    // Different archs can include different versions and therefore
 553    // different descriptions - so we need to check for both name and md5.
 554    string const Record = Pkg + ":" + MD5;
 555
 556    if (Included.find(Record) != Included.end())
 557       return true;
 558
 559    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 560            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 561
 562    Included.insert(Record);
 563    return true;
 564 }
 565                                                                         /*}}}*/
 566 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 567 // ---------------------------------------------------------------------
 568 /* */
 569 TranslationWriter::~TranslationWriter()
 570 {
 571    if (Comp == NULL)
 572       return;
 573
 574    delete Comp;
 575 }
 576                                                                         /*}}}*/
 577
 578 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* */
 581 SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
 582                              string const &ExtOverrides) :
 583    Db(DB), Stats(Db.Stats)
 584 {
 585    Output = stdout;
 586    AddPattern("*.dsc");
 587    DeLinkLimit = 0;
 588    Buffer = 0;
 589    BufSize = 0;
 590
 591    // Process the command line options
 592    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 593    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 594    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 595    DoSHA512 = _config->FindB("APT::FTPArchive::Sources::SHA512",DoSHA512);
 596    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 597    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 598
 599    // Read the override file
 600    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 601       return;
 602    else
 603       NoOverride = true;
 604
 605    // WTF?? The logic above: if we can't read binary overrides, don't even try
 606    // reading source overrides. if we can read binary overrides, then say there
 607    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 608
 609    if (ExtOverrides.empty() == false)
 610       SOver.ReadExtraOverride(ExtOverrides);
 611
 612    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 613       SOver.ReadOverride(SOverrides,true);
 614 }
 615                                                                         /*}}}*/
 616 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* */
 619 bool SourcesWriter::DoPackage(string FileName)
 620 {
 621    // Pull all the data we need form the DB
 622    if (Db.GetFileInfo(FileName,
 623                       false,  /* DoControl */
 624                       false,  /* DoContents */
 625                       false,  /* GenContentsOnly */
 626                       true,   /* DoSource */
 627                       DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false)
 628    {
 629       return false;
 630    }
 631
 632    // we need to perform a "write" here (this is what finish is doing)
 633    // because the call to Db.GetFileInfo() in the loop will change
 634    // the "db cursor"
 635    Db.Finish();
 636
 637    pkgTagSection Tags;
 638    if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
 639       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 640
 641    if (Tags.Exists("Source") == false)
 642       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
 643    Tags.Trim();
 644
 645    // Lookup the overide information, finding first the best priority.
 646    string BestPrio;
 647    string Bins = Tags.FindS("Binary");
 648    char Buffer[Bins.length() + 1];
 649    auto_ptr<Override::Item> OverItem(0);
 650    if (Bins.empty() == false)
 651    {
 652       strcpy(Buffer,Bins.c_str());
 653
 654       // Ignore too-long errors.
 655       char *BinList[400];
 656       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 657
 658       // Look at all the binaries
 659       unsigned char BestPrioV = pkgCache::State::Extra;
 660       for (unsigned I = 0; BinList[I] != 0; I++)
 661       {
 662          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 663          if (Itm.get() == 0)
 664             continue;
 665
 666          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 667          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 668          {
 669             BestPrioV = NewPrioV;
 670             BestPrio = Itm->Priority;
 671          }
 672
 673          if (OverItem.get() == 0)
 674             OverItem = Itm;
 675       }
 676    }
 677
 678    // If we need to do any rewriting of the header do it now..
 679    if (OverItem.get() == 0)
 680    {
 681       if (NoOverride == false)
 682       {
 683          NewLine(1);
 684          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 685       }
 686
 687       OverItem = auto_ptr<Override::Item>(new Override::Item);
 688    }
 689
 690    struct stat St;
 691    if (stat(FileName.c_str(), &St) != 0)
 692       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 693
 694    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 695    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 696    if (SOverItem.get() == 0)
 697    {
 698       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 699       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 700       if (SOverItem.get() == 0)
 701       {
 702         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 703          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 704          *SOverItem = *OverItem;
 705       }
 706    }
 707
 708    // Add the dsc to the files hash list
 709    string const strippedName = flNotDir(FileName);
 710    std::ostringstream ostreamFiles;
 711    if (DoMD5 == true && Tags.Exists("Files"))
 712       ostreamFiles << "\n " << Db.MD5Res.c_str() << " " << St.st_size << " "
 713                    << strippedName << "\n " << Tags.FindS("Files");
 714    string const Files = ostreamFiles.str();
 715
 716    std::ostringstream ostreamSha1;
 717    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 718       ostreamSha1 << "\n " << string(Db.SHA1Res.c_str()) << " " << St.st_size << " "
 719                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 720
 721    std::ostringstream ostreamSha256;
 722    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 723       ostreamSha256 << "\n " << string(Db.SHA256Res.c_str()) << " " << St.st_size << " "
 724                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 725
 726    std::ostringstream ostreamSha512;
 727    if (DoSHA512 == true && Tags.Exists("Checksums-Sha512"))
 728       ostreamSha512 << "\n " << string(Db.SHA512Res.c_str()) << " " << St.st_size << " "
 729                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 730
 731    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 732    string NewFileName;
 733    if (DirStrip.empty() == false &&
 734        FileName.length() > DirStrip.length() &&
 735        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 736       NewFileName = string(OriginalPath + DirStrip.length());
 737    else
 738       NewFileName = OriginalPath;
 739    if (PathPrefix.empty() == false)
 740       NewFileName = flCombine(PathPrefix,NewFileName);
 741
 742    string Directory = flNotFile(OriginalPath);
 743    string Package = Tags.FindS("Source");
 744
 745    // Perform operation over all of the files
 746    string ParseJnk;
 747    const char *C = Files.c_str();
 748    char *RealPath = NULL;
 749    for (;isspace(*C); C++);
 750    while (*C != 0)
 751    {
 752       // Parse each of the elements
 753       if (ParseQuoteWord(C,ParseJnk) == false ||
 754           ParseQuoteWord(C,ParseJnk) == false ||
 755           ParseQuoteWord(C,ParseJnk) == false)
 756          return _error->Error("Error parsing file record");
 757
 758       string OriginalPath = Directory + ParseJnk;
 759
 760       // Add missing hashes to source files
 761       if ((DoSHA1 == true && !Tags.Exists("Checksums-Sha1")) ||
 762           (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) ||
 763           (DoSHA512 == true && !Tags.Exists("Checksums-Sha512")))
 764       {
 765          if (Db.GetFileInfo(OriginalPath,
 766                             false, /* DoControl */
 767                             false, /* DoContents */
 768                             false, /* GenContentsOnly */
 769                             false, /* DoSource */
 770                             DoMD5, DoSHA1, DoSHA256, DoSHA512,
 771                             DoAlwaysStat) == false)
 772          {
 773             return _error->Error("Error getting file info");
 774          }
 775
 776          if (DoSHA1 == true && !Tags.Exists("Checksums-Sha1"))
 777             ostreamSha1 << "\n " << string(Db.SHA1Res) << " "
 778                << Db.GetFileSize() << " " << ParseJnk;
 779
 780          if (DoSHA256 == true && !Tags.Exists("Checksums-Sha256"))
 781             ostreamSha256 << "\n " << string(Db.SHA256Res) << " "
 782                << Db.GetFileSize() << " " << ParseJnk;
 783
 784          if (DoSHA512 == true && !Tags.Exists("Checksums-Sha512"))
 785             ostreamSha512 << "\n " << string(Db.SHA512Res) << " "
 786                << Db.GetFileSize() << " " << ParseJnk;
 787
 788          // write back the GetFileInfo() stats data
 789          Db.Finish();
 790       }
 791
 792       // Perform the delinking operation
 793       char Jnk[2];
 794
 795       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 796           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 797       {
 798          string RP = RealPath;
 799          free(RealPath);
 800          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 801             return false;
 802       }
 803    }
 804
 805    Directory = flNotFile(NewFileName);
 806    if (Directory.length() > 2)
 807       Directory.erase(Directory.end()-1);
 808
 809    string const ChecksumsSha1 = ostreamSha1.str();
 810    string const ChecksumsSha256 = ostreamSha256.str();
 811    string const ChecksumsSha512 = ostreamSha512.str();
 812
 813    // This lists all the changes to the fields we are going to make.
 814    // (5 hardcoded + checksums + maintainer + end marker)
 815    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 816
 817    unsigned int End = 0;
 818    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 819    if (Files.empty() == false)
 820       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 821    if (ChecksumsSha1.empty() == false)
 822       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 823    if (ChecksumsSha256.empty() == false)
 824       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 825    if (ChecksumsSha512.empty() == false)
 826       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 827    if (Directory != "./")
 828       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 829    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 830    SetTFRewriteData(Changes[End++],"Status",0);
 831
 832    // Rewrite the maintainer field if necessary
 833    bool MaintFailed;
 834    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 835    if (MaintFailed == true)
 836    {
 837       if (NoOverride == false)
 838       {
 839          NewLine(1);
 840          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 841                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 842       }
 843    }
 844    if (NewMaint.empty() == false)
 845       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 846
 847    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 848         I != SOverItem->FieldOverride.end(); ++I)
 849       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 850
 851    SetTFRewriteData(Changes[End++], 0, 0);
 852
 853    // Rewrite and store the fields.
 854    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 855       return false;
 856    fprintf(Output,"\n");
 857
 858    Stats.Packages++;
 859
 860    return true;
 861 }
 862                                                                         /*}}}*/
 863
 864 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 865 // ---------------------------------------------------------------------
 866 /* */
 867 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 868                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 869
 870 {
 871    SetExts(".deb");
 872    Output = stdout;
 873 }
 874                                                                         /*}}}*/
 875 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 876 // ---------------------------------------------------------------------
 877 /* If Package is the empty string the control record will be parsed to
 878    determine what the package name is. */
 879 bool ContentsWriter::DoPackage(string FileName, string Package)
 880 {
 881    if (!Db.GetFileInfo(FileName,
 882                        Package.empty(), /* DoControl */
 883                        true,            /* DoContents */
 884                        false,           /* GenContentsOnly */
 885                        false,           /* DoSource */
 886                        false,           /* DoMD5 */
 887                        false,           /* DoSHA1 */
 888                        false,           /* DoSHA256 */
 889                        false))          /* DoSHA512 */
 890    {
 891       return false;
 892    }
 893
 894    // Parse the package name
 895    if (Package.empty() == true)
 896    {
 897       Package = Db.Control.Section.FindS("Package");
 898    }
 899
 900    Db.Contents.Add(Gen,Package);
 901
 902    return Db.Finish();
 903 }
 904                                                                         /*}}}*/
 905 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 906 // ---------------------------------------------------------------------
 907 /* */
 908 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 909 {
 910    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 911    if (_error->PendingError() == true)
 912       return false;
 913
 914    // Open the package file
 915    FileFd Fd;
 916    if (Pkgs.OpenOld(Fd) == false)
 917       return false;
 918
 919    pkgTagFile Tags(&Fd);
 920    if (_error->PendingError() == true)
 921       return false;
 922
 923    // Parse.
 924    pkgTagSection Section;
 925    while (Tags.Step(Section) == true)
 926    {
 927       string File = flCombine(Prefix,Section.FindS("FileName"));
 928       string Package = Section.FindS("Section");
 929       if (Package.empty() == false && Package.end()[-1] != '/')
 930       {
 931          Package += '/';
 932          Package += Section.FindS("Package");
 933       }
 934       else
 935          Package += Section.FindS("Package");
 936
 937       DoPackage(File,Package);
 938       if (_error->empty() == false)
 939       {
 940          _error->Error("Errors apply to file '%s'",File.c_str());
 941          _error->DumpErrors();
 942       }
 943    }
 944
 945    // Tidy the compressor
 946    Fd.Close();
 947
 948    return true;
 949 }
 950
 951                                                                         /*}}}*/
 952
 953 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 954 // ---------------------------------------------------------------------
 955 /* */
 956 ReleaseWriter::ReleaseWriter(string const &/*DB*/)
 957 {
 958    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 959    {
 960       AddPattern("Packages");
 961       AddPattern("Packages.gz");
 962       AddPattern("Packages.bz2");
 963       AddPattern("Packages.lzma");
 964       AddPattern("Packages.xz");
 965       AddPattern("Translation-*");
 966       AddPattern("Sources");
 967       AddPattern("Sources.gz");
 968       AddPattern("Sources.bz2");
 969       AddPattern("Sources.lzma");
 970       AddPattern("Sources.xz");
 971       AddPattern("Release");
 972       AddPattern("Contents-*");
 973       AddPattern("Index");
 974       AddPattern("md5sum.txt");
 975    }
 976    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 977
 978    Output = stdout;
 979    time_t const now = time(NULL);
 980
 981    setlocale(LC_TIME, "C");
 982
 983    char datestr[128];
 984    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 985                 gmtime(&now)) == 0)
 986    {
 987       datestr[0] = '\0';
 988    }
 989
 990    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 991    char validstr[128];
 992    if (now == validuntil ||
 993        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 994                 gmtime(&validuntil)) == 0)
 995    {
 996       validstr[0] = '\0';
 997    }
 998
 999    setlocale(LC_TIME, "");
1000
1001    map<string,string> Fields;
1002    Fields["Origin"] = "";
1003    Fields["Label"] = "";
1004    Fields["Suite"] = "";
1005    Fields["Version"] = "";
1006    Fields["Codename"] = "";
1007    Fields["Date"] = datestr;
1008    Fields["Valid-Until"] = validstr;
1009    Fields["Architectures"] = "";
1010    Fields["Components"] = "";
1011    Fields["Description"] = "";
1012
1013    for(map<string,string>::const_iterator I = Fields.begin();
1014        I != Fields.end();
1015        ++I)
1016    {
1017       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1018       string Value = _config->Find(Config, (*I).second.c_str());
1019       if (Value == "")
1020          continue;
1021
1022       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1023    }
1024
1025    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1026    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1027    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1028 }
1029                                                                         /*}}}*/
1030 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1031 // ---------------------------------------------------------------------
1032 bool ReleaseWriter::DoPackage(string FileName)
1033 {
1034    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1035    string NewFileName;
1036    if (DirStrip.empty() == false &&
1037        FileName.length() > DirStrip.length() &&
1038        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1039                  DirStrip.begin(),DirStrip.end()) == 0)
1040    {
1041       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1042       while (NewFileName[0] == '/')
1043          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1044    }
1045    else
1046       NewFileName = FileName;
1047
1048    if (PathPrefix.empty() == false)
1049       NewFileName = flCombine(PathPrefix,NewFileName);
1050
1051    FileFd fd(FileName, FileFd::ReadOnly);
1052
1053    if (!fd.IsOpen())
1054    {
1055       return false;
1056    }
1057
1058    CheckSums[NewFileName].size = fd.Size();
1059
1060    Hashes hs;
1061    hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1062    if (DoMD5 == true)
1063       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1064    if (DoSHA1 == true)
1065       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1066    if (DoSHA256 == true)
1067       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1068    if (DoSHA512 == true)
1069       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1070    fd.Close();
1071
1072    return true;
1073 }
1074
1075                                                                         /*}}}*/
1076 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1077 // ---------------------------------------------------------------------
1078 void ReleaseWriter::Finish()
1079 {
1080    if (DoMD5 == true)
1081    {
1082       fprintf(Output, "MD5Sum:\n");
1083       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1084           I != CheckSums.end(); ++I)
1085       {
1086          fprintf(Output, " %s %16llu %s\n",
1087                  (*I).second.MD5.c_str(),
1088                  (*I).second.size,
1089                  (*I).first.c_str());
1090       }
1091    }
1092    if (DoSHA1 == true)
1093    {
1094       fprintf(Output, "SHA1:\n");
1095       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1096           I != CheckSums.end(); ++I)
1097       {
1098          fprintf(Output, " %s %16llu %s\n",
1099                  (*I).second.SHA1.c_str(),
1100                  (*I).second.size,
1101                  (*I).first.c_str());
1102       }
1103    }
1104    if (DoSHA256 == true)
1105    {
1106       fprintf(Output, "SHA256:\n");
1107       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1108           I != CheckSums.end(); ++I)
1109       {
1110          fprintf(Output, " %s %16llu %s\n",
1111                  (*I).second.SHA256.c_str(),
1112                  (*I).second.size,
1113                  (*I).first.c_str());
1114       }
1115    }
1116
1117    fprintf(Output, "SHA512:\n");
1118    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1119        I != CheckSums.end();
1120        ++I)
1121    {
1122       fprintf(Output, " %s %16llu %s\n",
1123               (*I).second.SHA512.c_str(),
1124               (*I).second.size,
1125               (*I).first.c_str());
1126    }
1127
1128 }