ftparchive/multicompress.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: multicompress.cc,v 1.4 2003/02/10 07:34:41 doogie Exp $
   4 /* ######################################################################
   5
   6    MultiCompressor
   7
   8    This class is very complicated in order to optimize for the common
   9    case of its use, writing a large set of compressed files that are
  10    different from the old set. It spawns off compressors in parallel
  11    to maximize compression throughput and has a separate task managing
  12    the data going into the compressors.
  13
  14    ##################################################################### */
  15                                                                         /*}}}*/
  16 // Include Files                                                        /*{{{*/
  17 #include <config.h>
  18
  19 #include <apt-pkg/fileutl.h>
  20 #include <apt-pkg/strutl.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/aptconfiguration.h>
  24 #include <apt-pkg/hashsum_template.h>
  25
  26 #include <ctype.h>
  27 #include <vector>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <sys/time.h>
  31 #include <unistd.h>
  32
  33 #include "multicompress.h"
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36
  37 using namespace std;
  38
  39
  40 // MultiCompress::MultiCompress - Constructor                           /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* Setup the file outputs, compression modes and fork the writer child */
  43 MultiCompress::MultiCompress(string const &Output,string const &Compress,
  44                              mode_t const &Permissions,bool const &Write) :
  45                         Permissions(Permissions)
  46 {
  47    Outputs = 0;
  48    Outputter = -1;
  49    UpdateMTime = 0;
  50
  51    /* Parse the compression string, a space separated lists of compresison
  52       types */
  53    string::const_iterator I = Compress.begin();
  54    for (; I != Compress.end();)
  55    {
  56       for (; I != Compress.end() && isspace(*I); ++I);
  57
  58       // Grab a word
  59       string::const_iterator Start = I;
  60       for (; I != Compress.end() && !isspace(*I); ++I);
  61
  62       // Find the matching compressor
  63       std::vector<APT::Configuration::Compressor> Compressors = APT::Configuration::getCompressors();
  64       std::vector<APT::Configuration::Compressor>::const_iterator Comp = Compressors.begin();
  65       for (; Comp != Compressors.end(); ++Comp)
  66          if (stringcmp(Start,I,Comp->Name.c_str()) == 0)
  67             break;
  68
  69       // Hmm.. unknown.
  70       if (Comp == Compressors.end())
  71       {
  72          _error->Warning(_("Unknown compression algorithm '%s'"),string(Start,I).c_str());
  73          continue;
  74       }
  75
  76       // Create and link in a new output
  77       Files *NewOut = new Files;
  78       NewOut->Next = Outputs;
  79       Outputs = NewOut;
  80       NewOut->CompressProg = *Comp;
  81       NewOut->Output = Output+Comp->Extension;
  82
  83       struct stat St;
  84       if (stat(NewOut->Output.c_str(),&St) == 0)
  85          NewOut->OldMTime = St.st_mtime;
  86       else
  87          NewOut->OldMTime = 0;
  88    }
  89
  90    if (Write == false)
  91        return;
  92
  93    /* Open all the temp files now so we can report any errors. File is
  94       made unreable to prevent people from touching it during creating. */
  95    for (Files *I = Outputs; I != 0; I = I->Next)
  96       I->TmpFile.Open(I->Output + ".new", FileFd::WriteOnly | FileFd::Create | FileFd::Empty, FileFd::Extension, 0600);
  97    if (_error->PendingError() == true)
  98       return;
  99
 100    if (Outputs == 0)
 101    {
 102       _error->Error(_("Compressed output %s needs a compression set"),Output.c_str());
 103       return;
 104    }
 105
 106    Start();
 107 }
 108                                                                         /*}}}*/
 109 // MultiCompress::~MultiCompress - Destructor                           /*{{{*/
 110 // ---------------------------------------------------------------------
 111 /* Just erase the file linked list. */
 112 MultiCompress::~MultiCompress()
 113 {
 114    Die();
 115
 116    for (; Outputs != 0;)
 117    {
 118       Files *Tmp = Outputs->Next;
 119       delete Outputs;
 120       Outputs = Tmp;
 121    }
 122 }
 123                                                                         /*}}}*/
 124 // MultiCompress::GetStat - Get stat information for compressed files   /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* This checks each compressed file to make sure it exists and returns
 127    stat information for a random file from the collection. False means
 128    one or more of the files is missing. */
 129 bool MultiCompress::GetStat(string const &Output,string const &Compress,struct stat &St)
 130 {
 131    /* Parse the compression string, a space separated lists of compresison
 132       types */
 133    string::const_iterator I = Compress.begin();
 134    bool DidStat = false;
 135    for (; I != Compress.end();)
 136    {
 137       for (; I != Compress.end() && isspace(*I); ++I);
 138
 139       // Grab a word
 140       string::const_iterator Start = I;
 141       for (; I != Compress.end() && !isspace(*I); ++I);
 142
 143       // Find the matching compressor
 144       std::vector<APT::Configuration::Compressor> Compressors = APT::Configuration::getCompressors();
 145       std::vector<APT::Configuration::Compressor>::const_iterator Comp = Compressors.begin();
 146       for (; Comp != Compressors.end(); ++Comp)
 147          if (stringcmp(Start,I,Comp->Name.c_str()) == 0)
 148             break;
 149
 150       // Hmm.. unknown.
 151       if (Comp == Compressors.end())
 152          continue;
 153
 154       string Name = Output+Comp->Extension;
 155       if (stat(Name.c_str(),&St) != 0)
 156          return false;
 157       DidStat = true;
 158    }
 159    return DidStat;
 160 }
 161                                                                         /*}}}*/
 162 // MultiCompress::Start - Start up the writer child                     /*{{{*/
 163 // ---------------------------------------------------------------------
 164 /* Fork a child and setup the communication pipe. */
 165 bool MultiCompress::Start()
 166 {
 167    // Create a data pipe
 168    int Pipe[2] = {-1,-1};
 169    if (pipe(Pipe) != 0)
 170       return _error->Errno("pipe",_("Failed to create IPC pipe to subprocess"));
 171    for (int I = 0; I != 2; I++)
 172       SetCloseExec(Pipe[I],true);
 173
 174    // The child..
 175    Outputter = fork();
 176    if (Outputter == 0)
 177    {
 178       close(Pipe[1]);
 179       Child(Pipe[0]);
 180       if (_error->PendingError() == true)
 181       {
 182          _error->DumpErrors();
 183          _exit(100);
 184       }
 185       _exit(0);
 186    };
 187
 188    close(Pipe[0]);
 189    if (Input.OpenDescriptor(Pipe[1], FileFd::WriteOnly, true) == false)
 190       return false;
 191
 192    if (Outputter == -1)
 193       return _error->Errno("fork",_("Failed to fork"));
 194    return true;
 195 }
 196                                                                         /*}}}*/
 197 // MultiCompress::Die - Clean up the writer                             /*{{{*/
 198 // ---------------------------------------------------------------------
 199 /* */
 200 bool MultiCompress::Die()
 201 {
 202    if (Input.IsOpen() == false)
 203       return true;
 204
 205    Input.Close();
 206    bool Res = ExecWait(Outputter,_("Compress child"),false);
 207    Outputter = -1;
 208    return Res;
 209 }
 210                                                                         /*}}}*/
 211 // MultiCompress::Finalize - Finish up writing                          /*{{{*/
 212 // ---------------------------------------------------------------------
 213 /* This is only necessary for statistics reporting. */
 214 bool MultiCompress::Finalize(unsigned long long &OutSize)
 215 {
 216    OutSize = 0;
 217    if (Input.IsOpen() == false || Die() == false)
 218       return false;
 219
 220    time_t Now;
 221    time(&Now);
 222
 223    // Check the mtimes to see if the files were replaced.
 224    bool Changed = false;
 225    for (Files *I = Outputs; I != 0; I = I->Next)
 226    {
 227       struct stat St;
 228       if (stat(I->Output.c_str(),&St) != 0)
 229          return  _error->Error(_("Internal error, failed to create %s"),
 230                                I->Output.c_str());
 231
 232       if (I->OldMTime != St.st_mtime)
 233          Changed = true;
 234       else
 235       {
 236          // Update the mtime if necessary
 237          if (UpdateMTime > 0 &&
 238              (Now - St.st_mtime > (signed)UpdateMTime || St.st_mtime > Now))
 239          {
 240             utimes(I->Output.c_str(), NULL);
 241             Changed = true;
 242          }
 243       }
 244
 245       // Force the file permissions
 246       if (St.st_mode != Permissions)
 247          chmod(I->Output.c_str(),Permissions);
 248
 249       OutSize += St.st_size;
 250    }
 251
 252    if (Changed == false)
 253       OutSize = 0;
 254
 255    return true;
 256 }
 257                                                                         /*}}}*/
 258 // MultiCompress::OpenOld - Open an old file                            /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This opens one of the original output files, possibly decompressing it. */
 261 bool MultiCompress::OpenOld(FileFd &Fd)
 262 {
 263    Files *Best = Outputs;
 264    for (Files *I = Outputs; I != 0; I = I->Next)
 265       if (Best->CompressProg.Cost > I->CompressProg.Cost)
 266          Best = I;
 267
 268    // Open the file
 269    return Fd.Open(Best->Output, FileFd::ReadOnly, FileFd::Extension);
 270 }
 271                                                                         /*}}}*/
 272 // MultiCompress::Child - The writer child                              /*{{{*/
 273 // ---------------------------------------------------------------------
 274 /* The child process forks a bunch of compression children and takes
 275    input on FD and passes it to all the compressor child. On the way it
 276    computes the MD5 of the raw data. After this the raw data in the
 277    original files is compared to see if this data is new. If the data
 278    is new then the temp files are renamed, otherwise they are erased. */
 279 bool MultiCompress::Child(int const &FD)
 280 {
 281    /* Okay, now we just feed data from FD to all the other FDs. Also
 282       stash a hash of the data to use later. */
 283    SetNonBlock(FD,false);
 284    unsigned char Buffer[32*1024];
 285    unsigned long long FileSize = 0;
 286    MD5Summation MD5;
 287    while (1)
 288    {
 289       WaitFd(FD,false);
 290       int Res = read(FD,Buffer,sizeof(Buffer));
 291       if (Res == 0)
 292          break;
 293       if (Res < 0)
 294          continue;
 295
 296       MD5.Add(Buffer,Res);
 297       FileSize += Res;
 298       for (Files *I = Outputs; I != 0; I = I->Next)
 299       {
 300          if (I->TmpFile.Write(Buffer, Res) == false)
 301          {
 302             _error->Errno("write",_("IO to subprocess/file failed"));
 303             break;
 304          }
 305       }
 306    }
 307
 308    if (_error->PendingError() == true)
 309       return false;
 310
 311    /* Now we have to copy the files over, or erase them if they
 312       have not changed. First find the cheapest decompressor */
 313    bool Missing = false;
 314    for (Files *I = Outputs; I != 0; I = I->Next)
 315    {
 316       if (I->OldMTime == 0)
 317       {
 318          Missing = true;
 319          break;
 320       }
 321    }
 322
 323    // Check the MD5 of the lowest cost entity.
 324    while (Missing == false)
 325    {
 326       FileFd CompFd;
 327       if (OpenOld(CompFd) == false)
 328       {
 329          _error->Discard();
 330          break;
 331       }
 332
 333       // Compute the hash
 334       MD5Summation OldMD5;
 335       unsigned long long NewFileSize = 0;
 336       while (1)
 337       {
 338          unsigned long long Res = 0;
 339          if (CompFd.Read(Buffer,sizeof(Buffer), &Res) == false)
 340             return _error->Errno("read",_("Failed to read while computing MD5"));
 341          if (Res == 0)
 342             break;
 343          NewFileSize += Res;
 344          OldMD5.Add(Buffer,Res);
 345       }
 346       CompFd.Close();
 347
 348       // Check the hash
 349       if (OldMD5.Result() == MD5.Result() &&
 350           FileSize == NewFileSize)
 351       {
 352          for (Files *I = Outputs; I != 0; I = I->Next)
 353          {
 354             I->TmpFile.Close();
 355             RemoveFile("MultiCompress::Child", I->TmpFile.Name());
 356          }
 357          return !_error->PendingError();
 358       }
 359       break;
 360    }
 361
 362    // Finalize
 363    for (Files *I = Outputs; I != 0; I = I->Next)
 364    {
 365       // Set the correct file modes
 366       chmod(I->TmpFile.Name().c_str(),Permissions);
 367
 368       if (rename(I->TmpFile.Name().c_str(),I->Output.c_str()) != 0)
 369          _error->Errno("rename",_("Failed to rename %s to %s"),
 370                        I->TmpFile.Name().c_str(),I->Output.c_str());
 371       I->TmpFile.Close();
 372    }
 373
 374    return !_error->PendingError();
 375 }
 376                                                                         /*}}}*/
 377