ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/deblistparser.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/fileutl.h>
  20 #include <apt-pkg/gpgv.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/debfile.h>
  25 #include <apt-pkg/pkgcache.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/sha2.h>
  28 #include <apt-pkg/tagfile.h>
  29
  30 #include <ctype.h>
  31 #include <fnmatch.h>
  32 #include <ftw.h>
  33 #include <locale.h>
  34 #include <string.h>
  35 #include <sys/stat.h>
  36 #include <sys/types.h>
  37 #include <unistd.h>
  38 #include <ctime>
  39 #include <iostream>
  40 #include <sstream>
  41 #include <memory>
  42 #include <utility>
  43
  44 #include "apt-ftparchive.h"
  45 #include "writer.h"
  46 #include "cachedb.h"
  47 #include "multicompress.h"
  48
  49 #include <apti18n.h>
  50                                                                         /*}}}*/
  51 using namespace std;
  52 FTWScanner *FTWScanner::Owner;
  53
  54 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  58                              const char *tag,
  59                              const char *rewrite,
  60                              const char *newtag = 0)
  61 {
  62     tfrd.Tag = tag;
  63     tfrd.Rewrite = rewrite;
  64     tfrd.NewTag = newtag;
  65 }
  66                                                                         /*}}}*/
  67
  68 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* */
  71 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  72 {
  73    ErrorPrinted = false;
  74    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  75
  76    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  77    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  78    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  79    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  80 }
  81                                                                         /*}}}*/
  82 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  83 // ---------------------------------------------------------------------
  84 /* This is the FTW scanner, it processes each directory element in the
  85    directory tree. */
  86 int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
  87 {
  88    if (Flag == FTW_DNR)
  89    {
  90       Owner->NewLine(1);
  91       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  92    }
  93    if (Flag == FTW_NS)
  94    {
  95       Owner->NewLine(1);
  96       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  97    }
  98    if (Flag != FTW_F)
  99       return 0;
 100
 101    return ScannerFile(File, true);
 102 }
 103                                                                         /*}}}*/
 104 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
 105 // ---------------------------------------------------------------------
 106 /* */
 107 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
 108 {
 109    const char *LastComponent = strrchr(File, '/');
 110    char *RealPath = NULL;
 111
 112    if (LastComponent == NULL)
 113       LastComponent = File;
 114    else
 115       LastComponent++;
 116
 117    vector<string>::const_iterator I;
 118    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 119    {
 120       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 121          break;
 122    }
 123    if (I == Owner->Patterns.end())
 124       return 0;
 125
 126    /* Process it. If the file is a link then resolve it into an absolute
 127       name.. This works best if the directory components the scanner are
 128       given are not links themselves. */
 129    char Jnk[2];
 130    Owner->OriginalPath = File;
 131    if (ReadLink &&
 132        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 133        (RealPath = realpath(File,NULL)) != 0)
 134    {
 135       Owner->DoPackage(RealPath);
 136       free(RealPath);
 137    }
 138    else
 139       Owner->DoPackage(File);
 140
 141    if (_error->empty() == false)
 142    {
 143       // Print any errors or warnings found
 144       string Err;
 145       bool SeenPath = false;
 146       while (_error->empty() == false)
 147       {
 148          Owner->NewLine(1);
 149
 150          bool const Type = _error->PopMessage(Err);
 151          if (Type == true)
 152             cerr << _("E: ") << Err << endl;
 153          else
 154             cerr << _("W: ") << Err << endl;
 155
 156          if (Err.find(File) != string::npos)
 157             SeenPath = true;
 158       }
 159
 160       if (SeenPath == false)
 161          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 162       return 0;
 163    }
 164
 165    return 0;
 166 }
 167                                                                         /*}}}*/
 168 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 169 // ---------------------------------------------------------------------
 170 /* */
 171 bool FTWScanner::RecursiveScan(string const &Dir)
 172 {
 173    char *RealPath = NULL;
 174    /* If noprefix is set then jam the scan root in, so we don't generate
 175       link followed paths out of control */
 176    if (InternalPrefix.empty() == true)
 177    {
 178       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 179          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 180       InternalPrefix = RealPath;
 181       free(RealPath);
 182    }
 183
 184    // Do recursive directory searching
 185    Owner = this;
 186    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 187
 188    // Error treewalking?
 189    if (Res != 0)
 190    {
 191       if (_error->PendingError() == false)
 192          _error->Errno("ftw",_("Tree walking failed"));
 193       return false;
 194    }
 195
 196    return true;
 197 }
 198                                                                         /*}}}*/
 199 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 200 // ---------------------------------------------------------------------
 201 /* This is an alternative to using FTW to locate files, it reads the list
 202    of files from another file. */
 203 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 204 {
 205    char *RealPath = NULL;
 206    /* If noprefix is set then jam the scan root in, so we don't generate
 207       link followed paths out of control */
 208    if (InternalPrefix.empty() == true)
 209    {
 210       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 211          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 212       InternalPrefix = RealPath;
 213       free(RealPath);
 214    }
 215
 216    Owner = this;
 217    FILE *List = fopen(File.c_str(),"r");
 218    if (List == 0)
 219       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 220
 221    /* We are a tad tricky here.. We prefix the buffer with the directory
 222       name, that way if we need a full path with just use line.. Sneaky and
 223       fully evil. */
 224    char Line[1000];
 225    char *FileStart;
 226    if (Dir.empty() == true || Dir.end()[-1] != '/')
 227       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 228    else
 229       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 230    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 231    {
 232       char *FileName = _strstrip(FileStart);
 233       if (FileName[0] == 0)
 234          continue;
 235
 236       if (FileName[0] != '/')
 237       {
 238          if (FileName != FileStart)
 239             memmove(FileStart,FileName,strlen(FileStart));
 240          FileName = Line;
 241       }
 242
 243 #if 0
 244       struct stat St;
 245       int Flag = FTW_F;
 246       if (stat(FileName,&St) != 0)
 247          Flag = FTW_NS;
 248 #endif
 249
 250       if (ScannerFile(FileName, false) != 0)
 251          break;
 252    }
 253
 254    fclose(List);
 255    return true;
 256 }
 257                                                                         /*}}}*/
 258 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* */
 261 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 262                         unsigned long long &DeLinkBytes,
 263                         unsigned long long const &FileSize)
 264 {
 265    // See if this isn't an internaly prefix'd file name.
 266    if (InternalPrefix.empty() == false &&
 267        InternalPrefix.length() < FileName.length() &&
 268        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 269                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 270    {
 271       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 272       {
 273          // Tidy up the display
 274          if (DeLinkBytes == 0)
 275             cout << endl;
 276
 277          NewLine(1);
 278          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 279                     SizeToStr(FileSize).c_str());
 280          c1out << flush;
 281
 282          if (NoLinkAct == false)
 283          {
 284             char OldLink[400];
 285             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 286                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 287             else
 288             {
 289                if (unlink(OriginalPath) != 0)
 290                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 291                else
 292                {
 293                   if (link(FileName.c_str(),OriginalPath) != 0)
 294                   {
 295                      // Panic! Restore the symlink
 296                      if (symlink(OldLink,OriginalPath) != 0)
 297                         _error->Errno("symlink", "failed to restore symlink");
 298                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 299                                           FileName.c_str(),
 300                                           OriginalPath);
 301                   }
 302                }
 303             }
 304          }
 305
 306          DeLinkBytes += FileSize;
 307          if (DeLinkBytes/1024 >= DeLinkLimit)
 308             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 309       }
 310
 311       FileName = OriginalPath;
 312    }
 313
 314    return true;
 315 }
 316                                                                         /*}}}*/
 317
 318 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 319 // ---------------------------------------------------------------------
 320 /* */
 321 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 322                                string const &Arch) :
 323    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 324 {
 325    Output = stdout;
 326    SetExts(".deb .udeb");
 327    DeLinkLimit = 0;
 328
 329    // Process the command line options
 330    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 331    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 332    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 333    DoSHA512 = _config->FindB("APT::FTPArchive::Packages::SHA512",DoSHA512);
 334    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 335    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 336    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 337    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 338
 339    if (Db.Loaded() == false)
 340       DoContents = false;
 341
 342    // Read the override file
 343    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 344       return;
 345    else
 346       NoOverride = true;
 347
 348    if (ExtOverrides.empty() == false)
 349       Over.ReadExtraOverride(ExtOverrides);
 350
 351    _error->DumpErrors();
 352 }
 353                                                                         /*}}}*/
 354 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* */
 357 bool FTWScanner::SetExts(string const &Vals)
 358 {
 359    ClearPatterns();
 360    string::size_type Start = 0;
 361    while (Start <= Vals.length()-1)
 362    {
 363       string::size_type const Space = Vals.find(' ',Start);
 364       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 365       if ( Arch.empty() == false )
 366       {
 367          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 368          AddPattern(string("*_all") + Vals.substr(Start, Length));
 369       }
 370       else
 371          AddPattern(string("*") + Vals.substr(Start, Length));
 372
 373       Start += Length + 1;
 374    }
 375
 376    return true;
 377 }
 378
 379                                                                         /*}}}*/
 380 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* This method takes a package and gets its control information and
 383    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 384    rewritten and the path/size/hash appended. */
 385 bool PackagesWriter::DoPackage(string FileName)
 386 {
 387    // Pull all the data we need form the DB
 388    if (Db.GetFileInfo(FileName,
 389                       true,   /* DoControl */
 390                       DoContents,
 391                       true,   /* GenContentsOnly */
 392                       false,  /* DoSource */
 393                       DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false)
 394    {
 395      return false;
 396    }
 397
 398    unsigned long long FileSize = Db.GetFileSize();
 399    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 400       return false;
 401
 402    // Lookup the overide information
 403    pkgTagSection &Tags = Db.Control.Section;
 404    string Package = Tags.FindS("Package");
 405    string Architecture;
 406    // if we generate a Packages file for a given arch, we use it to
 407    // look for overrides. if we run in "simple" mode without the
 408    // "Architecures" variable in the config we use the architecure value
 409    // from the deb file
 410    if(Arch != "")
 411       Architecture = Arch;
 412    else
 413       Architecture = Tags.FindS("Architecture");
 414    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 415
 416    if (Package.empty() == true)
 417       return _error->Error(_("Archive had no package field"));
 418
 419    // If we need to do any rewriting of the header do it now..
 420    if (OverItem.get() == 0)
 421    {
 422       if (NoOverride == false)
 423       {
 424          NewLine(1);
 425          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 426       }
 427
 428       OverItem = auto_ptr<Override::Item>(new Override::Item);
 429       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 430       OverItem->Priority = Tags.FindS("Priority");
 431    }
 432
 433    char Size[40];
 434    sprintf(Size,"%llu", (unsigned long long) FileSize);
 435
 436    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 437    string NewFileName;
 438    if (DirStrip.empty() == false &&
 439        FileName.length() > DirStrip.length() &&
 440        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 441                  DirStrip.begin(),DirStrip.end()) == 0)
 442       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 443    else
 444       NewFileName = FileName;
 445    if (PathPrefix.empty() == false)
 446       NewFileName = flCombine(PathPrefix,NewFileName);
 447
 448    /* Configuration says we don't want to include the long Description
 449       in the package file - instead we want to ship a separated file */
 450    string desc;
 451    if (LongDescription == false) {
 452       desc = Tags.FindS("Description").append("\n");
 453       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 454    }
 455
 456    // This lists all the changes to the fields we are going to make.
 457    // (7 hardcoded + maintainer + suggests + end marker)
 458    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 459
 460    unsigned int End = 0;
 461    SetTFRewriteData(Changes[End++], "Size", Size);
 462    if (DoMD5 == true)
 463       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 464    if (DoSHA1 == true)
 465       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 466    if (DoSHA256 == true)
 467       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 468    if (DoSHA512 == true)
 469       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 470    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 471    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 472    SetTFRewriteData(Changes[End++], "Status", 0);
 473    SetTFRewriteData(Changes[End++], "Optional", 0);
 474
 475    string DescriptionMd5;
 476    if (LongDescription == false) {
 477       MD5Summation descmd5;
 478       descmd5.Add(desc.c_str());
 479       DescriptionMd5 = descmd5.Result().Value();
 480       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 481       if (TransWriter != NULL)
 482          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 483    }
 484
 485    // Rewrite the maintainer field if necessary
 486    bool MaintFailed;
 487    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 488    if (MaintFailed == true)
 489    {
 490       if (NoOverride == false)
 491       {
 492          NewLine(1);
 493          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 494                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 495       }
 496    }
 497
 498    if (NewMaint.empty() == false)
 499       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 500
 501    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 502       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 503       but dpkg does this append bit. So we do the append bit, at least that way the
 504       status file and package file will remain similar. There are other transforms
 505       but optional is the only legacy one still in use for some lazy reason. */
 506    string OptionalStr = Tags.FindS("Optional");
 507    if (OptionalStr.empty() == false)
 508    {
 509       if (Tags.FindS("Suggests").empty() == false)
 510          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 511       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 512    }
 513
 514    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 515         I != OverItem->FieldOverride.end(); ++I)
 516       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 517
 518    SetTFRewriteData(Changes[End++], 0, 0);
 519
 520    // Rewrite and store the fields.
 521    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 522       return false;
 523    fprintf(Output,"\n");
 524
 525    return Db.Finish();
 526 }
 527                                                                         /*}}}*/
 528
 529 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* Create a Translation-Master file for this Packages file */
 532 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 533                                         mode_t const &Permissions) : Output(NULL),
 534                                                         RefCounter(0)
 535 {
 536    if (File.empty() == true)
 537       return;
 538
 539    Comp = new MultiCompress(File, TransCompress, Permissions);
 540    Output = Comp->Input;
 541 }
 542                                                                         /*}}}*/
 543 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* Create a Translation-Master file for this Packages file */
 546 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 547                                   string const &MD5)
 548 {
 549    if (Output == NULL)
 550       return true;
 551
 552    // Different archs can include different versions and therefore
 553    // different descriptions - so we need to check for both name and md5.
 554    string const Record = Pkg + ":" + MD5;
 555
 556    if (Included.find(Record) != Included.end())
 557       return true;
 558
 559    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 560            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 561
 562    Included.insert(Record);
 563    return true;
 564 }
 565                                                                         /*}}}*/
 566 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 567 // ---------------------------------------------------------------------
 568 /* */
 569 TranslationWriter::~TranslationWriter()
 570 {
 571    if (Comp == NULL)
 572       return;
 573
 574    delete Comp;
 575 }
 576                                                                         /*}}}*/
 577
 578 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* */
 581 SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
 582                              string const &ExtOverrides) :
 583    Db(DB), Stats(Db.Stats)
 584 {
 585    Output = stdout;
 586    AddPattern("*.dsc");
 587    DeLinkLimit = 0;
 588    Buffer = 0;
 589    BufSize = 0;
 590
 591    // Process the command line options
 592    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 593    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 594    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 595    DoSHA512 = _config->FindB("APT::FTPArchive::Sources::SHA512",DoSHA512);
 596    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 597    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 598
 599    // Read the override file
 600    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 601       return;
 602    else
 603       NoOverride = true;
 604
 605    // WTF?? The logic above: if we can't read binary overrides, don't even try
 606    // reading source overrides. if we can read binary overrides, then say there
 607    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 608
 609    if (ExtOverrides.empty() == false)
 610       SOver.ReadExtraOverride(ExtOverrides);
 611
 612    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 613       SOver.ReadOverride(SOverrides,true);
 614 }
 615                                                                         /*}}}*/
 616 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* */
 619 bool SourcesWriter::DoPackage(string FileName)
 620 {
 621    // Pull all the data we need form the DB
 622    if (Db.GetFileInfo(FileName,
 623                       false,  /* DoControl */
 624                       false,  /* DoContents */
 625                       false,  /* GenContentsOnly */
 626                       true,   /* DoSource */
 627                       DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false)
 628    {
 629       return false;
 630    }
 631
 632    // we need to perform a "write" here (this is what finish is doing)
 633    // because the call to Db.GetFileInfo() in the loop will change
 634    // the "db cursor"
 635    Db.Finish();
 636
 637    // read stuff
 638    char *Start = Db.Dsc.Data;
 639    char *BlkEnd = Db.Dsc.Data + Db.Dsc.Length;
 640
 641    // Add extra \n to the end, just in case (as in clearsigned they are missing)
 642    *BlkEnd++ = '\n';
 643    *BlkEnd++ = '\n';
 644
 645    pkgTagSection Tags;
 646    if (Tags.Scan(Start,BlkEnd - Start) == false)
 647       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 648
 649    if (Tags.Exists("Source") == false)
 650       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
 651    Tags.Trim();
 652
 653    // Lookup the overide information, finding first the best priority.
 654    string BestPrio;
 655    string Bins = Tags.FindS("Binary");
 656    char Buffer[Bins.length() + 1];
 657    auto_ptr<Override::Item> OverItem(0);
 658    if (Bins.empty() == false)
 659    {
 660       strcpy(Buffer,Bins.c_str());
 661
 662       // Ignore too-long errors.
 663       char *BinList[400];
 664       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 665
 666       // Look at all the binaries
 667       unsigned char BestPrioV = pkgCache::State::Extra;
 668       for (unsigned I = 0; BinList[I] != 0; I++)
 669       {
 670          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 671          if (Itm.get() == 0)
 672             continue;
 673
 674          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 675          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 676          {
 677             BestPrioV = NewPrioV;
 678             BestPrio = Itm->Priority;
 679          }
 680
 681          if (OverItem.get() == 0)
 682             OverItem = Itm;
 683       }
 684    }
 685
 686    // If we need to do any rewriting of the header do it now..
 687    if (OverItem.get() == 0)
 688    {
 689       if (NoOverride == false)
 690       {
 691          NewLine(1);
 692          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 693       }
 694
 695       OverItem = auto_ptr<Override::Item>(new Override::Item);
 696    }
 697
 698    struct stat St;
 699    if (stat(FileName.c_str(), &St) != 0)
 700       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 701
 702    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 703    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 704    if (SOverItem.get() == 0)
 705    {
 706       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 707       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 708       if (SOverItem.get() == 0)
 709       {
 710         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 711          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 712          *SOverItem = *OverItem;
 713       }
 714    }
 715
 716    // Add the dsc to the files hash list
 717    string const strippedName = flNotDir(FileName);
 718    std::ostringstream ostreamFiles;
 719    if (DoMD5 == true && Tags.Exists("Files"))
 720       ostreamFiles << "\n " << Db.MD5Res.c_str() << " " << St.st_size << " "
 721                    << strippedName << "\n " << Tags.FindS("Files");
 722    string const Files = ostreamFiles.str();
 723
 724    std::ostringstream ostreamSha1;
 725    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 726       ostreamSha1 << "\n " << string(Db.SHA1Res.c_str()) << " " << St.st_size << " "
 727                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 728
 729    std::ostringstream ostreamSha256;
 730    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 731       ostreamSha256 << "\n " << string(Db.SHA256Res.c_str()) << " " << St.st_size << " "
 732                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 733
 734    std::ostringstream ostreamSha512;
 735    if (DoSHA512 == true && Tags.Exists("Checksums-Sha512"))
 736       ostreamSha512 << "\n " << string(Db.SHA512Res.c_str()) << " " << St.st_size << " "
 737                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 738
 739    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 740    string NewFileName;
 741    if (DirStrip.empty() == false &&
 742        FileName.length() > DirStrip.length() &&
 743        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 744       NewFileName = string(OriginalPath + DirStrip.length());
 745    else
 746       NewFileName = OriginalPath;
 747    if (PathPrefix.empty() == false)
 748       NewFileName = flCombine(PathPrefix,NewFileName);
 749
 750    string Directory = flNotFile(OriginalPath);
 751    string Package = Tags.FindS("Source");
 752
 753    // Perform operation over all of the files
 754    string ParseJnk;
 755    const char *C = Files.c_str();
 756    char *RealPath = NULL;
 757    for (;isspace(*C); C++);
 758    while (*C != 0)
 759    {
 760       // Parse each of the elements
 761       if (ParseQuoteWord(C,ParseJnk) == false ||
 762           ParseQuoteWord(C,ParseJnk) == false ||
 763           ParseQuoteWord(C,ParseJnk) == false)
 764          return _error->Error("Error parsing file record");
 765
 766       string OriginalPath = Directory + ParseJnk;
 767
 768       // Add missing hashes to source files
 769       if ((DoSHA1 == true && !Tags.Exists("Checksums-Sha1")) ||
 770           (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) ||
 771           (DoSHA512 == true && !Tags.Exists("Checksums-Sha512")))
 772       {
 773          if (Db.GetFileInfo(OriginalPath,
 774                             false, /* DoControl */
 775                             false, /* DoContents */
 776                             false, /* GenContentsOnly */
 777                             false, /* DoSource */
 778                             DoMD5, DoSHA1, DoSHA256, DoSHA512,
 779                             DoAlwaysStat) == false)
 780          {
 781             return _error->Error("Error getting file info");
 782          }
 783
 784          if (DoSHA1 == true && !Tags.Exists("Checksums-Sha1"))
 785             ostreamSha1 << "\n " << string(Db.SHA1Res) << " "
 786                << Db.GetFileSize() << " " << ParseJnk;
 787
 788          if (DoSHA256 == true && !Tags.Exists("Checksums-Sha256"))
 789             ostreamSha256 << "\n " << string(Db.SHA256Res) << " "
 790                << Db.GetFileSize() << " " << ParseJnk;
 791
 792          if (DoSHA512 == true && !Tags.Exists("Checksums-Sha512"))
 793             ostreamSha512 << "\n " << string(Db.SHA512Res) << " "
 794                << Db.GetFileSize() << " " << ParseJnk;
 795
 796          // write back the GetFileInfo() stats data
 797          Db.Finish();
 798       }
 799
 800       // Perform the delinking operation
 801       char Jnk[2];
 802
 803       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 804           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 805       {
 806          string RP = RealPath;
 807          free(RealPath);
 808          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 809             return false;
 810       }
 811    }
 812
 813    Directory = flNotFile(NewFileName);
 814    if (Directory.length() > 2)
 815       Directory.erase(Directory.end()-1);
 816
 817    string const ChecksumsSha1 = ostreamSha1.str();
 818    string const ChecksumsSha256 = ostreamSha256.str();
 819    string const ChecksumsSha512 = ostreamSha512.str();
 820
 821    // This lists all the changes to the fields we are going to make.
 822    // (5 hardcoded + checksums + maintainer + end marker)
 823    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 824
 825    unsigned int End = 0;
 826    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 827    if (Files.empty() == false)
 828       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 829    if (ChecksumsSha1.empty() == false)
 830       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 831    if (ChecksumsSha256.empty() == false)
 832       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 833    if (ChecksumsSha512.empty() == false)
 834       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 835    if (Directory != "./")
 836       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 837    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 838    SetTFRewriteData(Changes[End++],"Status",0);
 839
 840    // Rewrite the maintainer field if necessary
 841    bool MaintFailed;
 842    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 843    if (MaintFailed == true)
 844    {
 845       if (NoOverride == false)
 846       {
 847          NewLine(1);
 848          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 849                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 850       }
 851    }
 852    if (NewMaint.empty() == false)
 853       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 854
 855    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 856         I != SOverItem->FieldOverride.end(); ++I)
 857       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 858
 859    SetTFRewriteData(Changes[End++], 0, 0);
 860
 861    // Rewrite and store the fields.
 862    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 863       return false;
 864    fprintf(Output,"\n");
 865
 866    Stats.Packages++;
 867
 868    return true;
 869 }
 870                                                                         /*}}}*/
 871
 872 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 873 // ---------------------------------------------------------------------
 874 /* */
 875 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 876                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 877
 878 {
 879    SetExts(".deb");
 880    Output = stdout;
 881 }
 882                                                                         /*}}}*/
 883 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 884 // ---------------------------------------------------------------------
 885 /* If Package is the empty string the control record will be parsed to
 886    determine what the package name is. */
 887 bool ContentsWriter::DoPackage(string FileName, string Package)
 888 {
 889    if (!Db.GetFileInfo(FileName,
 890                        Package.empty(), /* DoControl */
 891                        true,            /* DoContents */
 892                        false,           /* GenContentsOnly */
 893                        false,           /* DoSource */
 894                        false,           /* DoMD5 */
 895                        false,           /* DoSHA1 */
 896                        false,           /* DoSHA256 */
 897                        false))          /* DoSHA512 */
 898    {
 899       return false;
 900    }
 901
 902    // Parse the package name
 903    if (Package.empty() == true)
 904    {
 905       Package = Db.Control.Section.FindS("Package");
 906    }
 907
 908    Db.Contents.Add(Gen,Package);
 909
 910    return Db.Finish();
 911 }
 912                                                                         /*}}}*/
 913 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 914 // ---------------------------------------------------------------------
 915 /* */
 916 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 917 {
 918    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 919    if (_error->PendingError() == true)
 920       return false;
 921
 922    // Open the package file
 923    FileFd Fd;
 924    if (Pkgs.OpenOld(Fd) == false)
 925       return false;
 926
 927    pkgTagFile Tags(&Fd);
 928    if (_error->PendingError() == true)
 929       return false;
 930
 931    // Parse.
 932    pkgTagSection Section;
 933    while (Tags.Step(Section) == true)
 934    {
 935       string File = flCombine(Prefix,Section.FindS("FileName"));
 936       string Package = Section.FindS("Section");
 937       if (Package.empty() == false && Package.end()[-1] != '/')
 938       {
 939          Package += '/';
 940          Package += Section.FindS("Package");
 941       }
 942       else
 943          Package += Section.FindS("Package");
 944
 945       DoPackage(File,Package);
 946       if (_error->empty() == false)
 947       {
 948          _error->Error("Errors apply to file '%s'",File.c_str());
 949          _error->DumpErrors();
 950       }
 951    }
 952
 953    // Tidy the compressor
 954    Fd.Close();
 955
 956    return true;
 957 }
 958
 959                                                                         /*}}}*/
 960
 961 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 962 // ---------------------------------------------------------------------
 963 /* */
 964 ReleaseWriter::ReleaseWriter(string const &/*DB*/)
 965 {
 966    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 967    {
 968       AddPattern("Packages");
 969       AddPattern("Packages.gz");
 970       AddPattern("Packages.bz2");
 971       AddPattern("Packages.lzma");
 972       AddPattern("Packages.xz");
 973       AddPattern("Translation-*");
 974       AddPattern("Sources");
 975       AddPattern("Sources.gz");
 976       AddPattern("Sources.bz2");
 977       AddPattern("Sources.lzma");
 978       AddPattern("Sources.xz");
 979       AddPattern("Release");
 980       AddPattern("Contents-*");
 981       AddPattern("Index");
 982       AddPattern("md5sum.txt");
 983    }
 984    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 985
 986    Output = stdout;
 987    time_t const now = time(NULL);
 988
 989    setlocale(LC_TIME, "C");
 990
 991    char datestr[128];
 992    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 993                 gmtime(&now)) == 0)
 994    {
 995       datestr[0] = '\0';
 996    }
 997
 998    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 999    char validstr[128];
1000    if (now == validuntil ||
1001        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
1002                 gmtime(&validuntil)) == 0)
1003    {
1004       validstr[0] = '\0';
1005    }
1006
1007    setlocale(LC_TIME, "");
1008
1009    map<string,string> Fields;
1010    Fields["Origin"] = "";
1011    Fields["Label"] = "";
1012    Fields["Suite"] = "";
1013    Fields["Version"] = "";
1014    Fields["Codename"] = "";
1015    Fields["Date"] = datestr;
1016    Fields["Valid-Until"] = validstr;
1017    Fields["Architectures"] = "";
1018    Fields["Components"] = "";
1019    Fields["Description"] = "";
1020
1021    for(map<string,string>::const_iterator I = Fields.begin();
1022        I != Fields.end();
1023        ++I)
1024    {
1025       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1026       string Value = _config->Find(Config, (*I).second.c_str());
1027       if (Value == "")
1028          continue;
1029
1030       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1031    }
1032
1033    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1034    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1035    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1036 }
1037                                                                         /*}}}*/
1038 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1039 // ---------------------------------------------------------------------
1040 bool ReleaseWriter::DoPackage(string FileName)
1041 {
1042    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1043    string NewFileName;
1044    if (DirStrip.empty() == false &&
1045        FileName.length() > DirStrip.length() &&
1046        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1047                  DirStrip.begin(),DirStrip.end()) == 0)
1048    {
1049       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1050       while (NewFileName[0] == '/')
1051          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1052    }
1053    else
1054       NewFileName = FileName;
1055
1056    if (PathPrefix.empty() == false)
1057       NewFileName = flCombine(PathPrefix,NewFileName);
1058
1059    FileFd fd(FileName, FileFd::ReadOnly);
1060
1061    if (!fd.IsOpen())
1062    {
1063       return false;
1064    }
1065
1066    CheckSums[NewFileName].size = fd.Size();
1067
1068    Hashes hs;
1069    hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1070    if (DoMD5 == true)
1071       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1072    if (DoSHA1 == true)
1073       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1074    if (DoSHA256 == true)
1075       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1076    if (DoSHA512 == true)
1077       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1078    fd.Close();
1079
1080    return true;
1081 }
1082
1083                                                                         /*}}}*/
1084 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1085 // ---------------------------------------------------------------------
1086 void ReleaseWriter::Finish()
1087 {
1088    if (DoMD5 == true)
1089    {
1090       fprintf(Output, "MD5Sum:\n");
1091       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1092           I != CheckSums.end(); ++I)
1093       {
1094          fprintf(Output, " %s %16llu %s\n",
1095                  (*I).second.MD5.c_str(),
1096                  (*I).second.size,
1097                  (*I).first.c_str());
1098       }
1099    }
1100    if (DoSHA1 == true)
1101    {
1102       fprintf(Output, "SHA1:\n");
1103       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1104           I != CheckSums.end(); ++I)
1105       {
1106          fprintf(Output, " %s %16llu %s\n",
1107                  (*I).second.SHA1.c_str(),
1108                  (*I).second.size,
1109                  (*I).first.c_str());
1110       }
1111    }
1112    if (DoSHA256 == true)
1113    {
1114       fprintf(Output, "SHA256:\n");
1115       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1116           I != CheckSums.end(); ++I)
1117       {
1118          fprintf(Output, " %s %16llu %s\n",
1119                  (*I).second.SHA256.c_str(),
1120                  (*I).second.size,
1121                  (*I).first.c_str());
1122       }
1123    }
1124
1125    fprintf(Output, "SHA512:\n");
1126    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1127        I != CheckSums.end();
1128        ++I)
1129    {
1130       fprintf(Output, " %s %16llu %s\n",
1131               (*I).second.SHA512.c_str(),
1132               (*I).second.size,
1133               (*I).first.c_str());
1134    }
1135
1136 }