Add support for writing by-hash dirs in apt-ftparchive

[apt.git] / ftparchive / multicompress.cc

diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc
index 04560f4ab0e87a2fec5487a1fa2973167a7318ed..08a3cff5a97abc3342e35b1bbba4f9c417e05099 100644 (file)
--- a/ftparchive/multicompress.cc
+++ b/ftparchive/multicompress.cc
@@ -16,15 +16,19 @@
 // Include Files                                                       /*{{{*/
 #include <config.h>
 
 // Include Files                                                       /*{{{*/
 #include <config.h>
 

+#include <apt-pkg/fileutl.h>

 #include <apt-pkg/strutl.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/md5.h>
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/md5.h>

+#include <apt-pkg/aptconfiguration.h>
+#include <apt-pkg/hashsum_template.h>

 
 

+#include <ctype.h>
+#include <vector>

 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/stat.h>

-#include <utime.h>
+#include <sys/time.h>

 #include <unistd.h>
 #include <unistd.h>

-#include <iostream>

 
 #include "multicompress.h"
 #include <apti18n.h>
 
 #include "multicompress.h"
 #include <apti18n.h>


@@ -42,7 +46,6 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress,
 {
    Outputs = 0;
    Outputter = -1;
 {
    Outputs = 0;
    Outputter = -1;

-   Input = 0;

    UpdateMTime = 0;
 
    /* Parse the compression string, a space separated lists of compresison
    UpdateMTime = 0;
 
    /* Parse the compression string, a space separated lists of compresison


@@ -50,11 +53,11 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress,
    string::const_iterator I = Compress.begin();
    for (; I != Compress.end();)
    {
    string::const_iterator I = Compress.begin();
    for (; I != Compress.end();)
    {

-      for (; I != Compress.end() && isspace(*I); I++);
+      for (; I != Compress.end() && isspace(*I); ++I);

       
       // Grab a word
       string::const_iterator Start = I;
       
       // Grab a word
       string::const_iterator Start = I;

-      for (; I != Compress.end() && !isspace(*I); I++);
+      for (; I != Compress.end() && !isspace(*I); ++I);

 
       // Find the matching compressor
       std::vector<APT::Configuration::Compressor> Compressors = APT::Configuration::getCompressors();
 
       // Find the matching compressor
       std::vector<APT::Configuration::Compressor> Compressors = APT::Configuration::getCompressors();


@@ -90,7 +93,7 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress,
    /* Open all the temp files now so we can report any errors. File is 
       made unreable to prevent people from touching it during creating. */
    for (Files *I = Outputs; I != 0; I = I->Next)
    /* Open all the temp files now so we can report any errors. File is 
       made unreable to prevent people from touching it during creating. */
    for (Files *I = Outputs; I != 0; I = I->Next)

-      I->TmpFile.Open(I->Output + ".new",FileFd::WriteEmpty,0600);
+      I->TmpFile.Open(I->Output + ".new", FileFd::WriteOnly | FileFd::Create | FileFd::Empty, FileFd::Extension, 0600);

    if (_error->PendingError() == true)
       return;
 
    if (_error->PendingError() == true)
       return;
 


@@ -131,11 +134,11 @@ bool MultiCompress::GetStat(string const &Output,string const &Compress,struct s
    bool DidStat = false;
    for (; I != Compress.end();)
    {
    bool DidStat = false;
    for (; I != Compress.end();)
    {

-      for (; I != Compress.end() && isspace(*I); I++);
+      for (; I != Compress.end() && isspace(*I); ++I);

       
       // Grab a word
       string::const_iterator Start = I;
       
       // Grab a word
       string::const_iterator Start = I;

-      for (; I != Compress.end() && !isspace(*I); I++);
+      for (; I != Compress.end() && !isspace(*I); ++I);

 
       // Find the matching compressor
       std::vector<APT::Configuration::Compressor> Compressors = APT::Configuration::getCompressors();
 
       // Find the matching compressor
       std::vector<APT::Configuration::Compressor> Compressors = APT::Configuration::getCompressors();


@@ -182,18 +185,12 @@ bool MultiCompress::Start()
       _exit(0);
    };
 
       _exit(0);
    };
 

-   /* Tidy up the temp files, we open them in the constructor so as to
-      get proper error reporting. Close them now. */
-   for (Files *I = Outputs; I != 0; I = I->Next)
-      I->TmpFile.Close();
-   

    close(Pipe[0]);
    close(Pipe[0]);

-   Input = fdopen(Pipe[1],"w");
-   if (Input == 0)
-      return _error->Errno("fdopen",_("Failed to create FILE*"));
-   
+   if (Input.OpenDescriptor(Pipe[1], FileFd::WriteOnly, true) == false)
+      return false;
+

    if (Outputter == -1)
    if (Outputter == -1)

-      return _error->Errno("fork",_("Failed to fork"));   
+      return _error->Errno("fork",_("Failed to fork"));

    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/


@@ -202,11 +199,10 @@ bool MultiCompress::Start()
 /* */
 bool MultiCompress::Die()
 {
 /* */
 bool MultiCompress::Die()
 {

-   if (Input == 0)
+   if (Input.IsOpen() == false)

       return true;
       return true;

-   
-   fclose(Input);
-   Input = 0;
+
+   Input.Close();

    bool Res = ExecWait(Outputter,_("Compress child"),false);
    Outputter = -1;
    return Res;
    bool Res = ExecWait(Outputter,_("Compress child"),false);
    Outputter = -1;
    return Res;


@@ -215,10 +211,10 @@ bool MultiCompress::Die()
 // MultiCompress::Finalize - Finish up writing                         /*{{{*/
 // ---------------------------------------------------------------------
 /* This is only necessary for statistics reporting. */
 // MultiCompress::Finalize - Finish up writing                         /*{{{*/
 // ---------------------------------------------------------------------
 /* This is only necessary for statistics reporting. */

-bool MultiCompress::Finalize(unsigned long &OutSize)
+bool MultiCompress::Finalize(unsigned long long &OutSize)

 {
    OutSize = 0;
 {
    OutSize = 0;

-   if (Input == 0 || Die() == false)
+   if (Input.IsOpen() == false || Die() == false)

       return false;
    
    time_t Now;
       return false;
    
    time_t Now;


@@ -238,14 +234,12 @@ bool MultiCompress::Finalize(unsigned long &OutSize)
       else
       {
         // Update the mtime if necessary
       else
       {
         // Update the mtime if necessary

-        if (UpdateMTime > 0 && 
+        if (UpdateMTime > 0 &&

             (Now - St.st_mtime > (signed)UpdateMTime || St.st_mtime > Now))
         {
             (Now - St.st_mtime > (signed)UpdateMTime || St.st_mtime > Now))
         {

-           struct utimbuf Buf;
-           Buf.actime = Buf.modtime = Now;
-           utime(I->Output.c_str(),&Buf);
+           utimes(I->Output.c_str(), NULL);

            Changed = true;
            Changed = true;

-        }           
+        }

       }
       
       // Force the file permissions
       }
       
       // Force the file permissions


@@ -261,77 +255,10 @@ bool MultiCompress::Finalize(unsigned long &OutSize)
    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/

-// MultiCompress::OpenCompress - Open the compressor                   /*{{{*/
-// ---------------------------------------------------------------------
-/* This opens the compressor, either in compress mode or decompress 
-   mode. FileFd is always the compressor input/output file, 
-   OutFd is the created pipe, Input for Compress, Output for Decompress. */
-bool MultiCompress::OpenCompress(APT::Configuration::Compressor const &Prog,
-                                pid_t &Pid,int const &FileFd,int &OutFd,bool const &Comp)
-{
-   Pid = -1;
-   
-   // No compression
-   if (Prog.Binary.empty() == true)
-   {
-      OutFd = dup(FileFd);
-      return true;
-   }
-      
-   // Create a data pipe
-   int Pipe[2] = {-1,-1};
-   if (pipe(Pipe) != 0)
-      return _error->Errno("pipe",_("Failed to create subprocess IPC"));
-   for (int J = 0; J != 2; J++)
-      SetCloseExec(Pipe[J],true);
-
-   if (Comp == true)
-      OutFd = Pipe[1];
-   else
-      OutFd = Pipe[0];
-   
-   // The child..
-   Pid = ExecFork();
-   if (Pid == 0)
-   {
-      if (Comp == true)
-      {
-        dup2(FileFd,STDOUT_FILENO);
-        dup2(Pipe[0],STDIN_FILENO);
-      }   
-      else
-      {
-        dup2(FileFd,STDIN_FILENO);
-        dup2(Pipe[1],STDOUT_FILENO);
-      }
-      
-      SetCloseExec(STDOUT_FILENO,false);
-      SetCloseExec(STDIN_FILENO,false);
-
-      std::vector<char const*> Args;
-      Args.push_back(Prog.Binary.c_str());
-      std::vector<std::string> const * const addArgs =
-               (Comp == true) ? &(Prog.CompressArgs) : &(Prog.UncompressArgs);
-      for (std::vector<std::string>::const_iterator a = addArgs->begin();
-          a != addArgs->end(); ++a)
-        Args.push_back(a->c_str());
-      Args.push_back(NULL);
-
-      execvp(Args[0],(char **)&Args[0]);
-      cerr << _("Failed to exec compressor ") << Args[0] << endl;
-      _exit(100);
-   };      
-   if (Comp == true)
-      close(Pipe[0]);
-   else
-      close(Pipe[1]);
-   return true;
-}
-                                                                       /*}}}*/

 // MultiCompress::OpenOld - Open an old file                           /*{{{*/
 // ---------------------------------------------------------------------
 /* This opens one of the original output files, possibly decompressing it. */
 // MultiCompress::OpenOld - Open an old file                           /*{{{*/
 // ---------------------------------------------------------------------
 /* This opens one of the original output files, possibly decompressing it. */

-bool MultiCompress::OpenOld(int &Fd,pid_t &Proc)
+bool MultiCompress::OpenOld(FileFd &Fd)

 {
    Files *Best = Outputs;
    for (Files *I = Outputs; I != 0; I = I->Next)
 {
    Files *Best = Outputs;
    for (Files *I = Outputs; I != 0; I = I->Next)


@@ -339,29 +266,9 @@ bool MultiCompress::OpenOld(int &Fd,pid_t &Proc)
         Best = I;
 
    // Open the file
         Best = I;
 
    // Open the file

-   FileFd F(Best->Output,FileFd::ReadOnly);
-   if (_error->PendingError() == true)
-      return false;
-   
-   // Decompress the file so we can read it
-   if (OpenCompress(Best->CompressProg,Proc,F.Fd(),Fd,false) == false)
-      return false;
-   
-   return true;
+   return Fd.Open(Best->Output, FileFd::ReadOnly, FileFd::Extension);

 }
                                                                        /*}}}*/
 }
                                                                        /*}}}*/

-// MultiCompress::CloseOld - Close the old file                                /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool MultiCompress::CloseOld(int Fd,pid_t Proc)
-{
-   close(Fd);
-   if (Proc != -1)
-      if (ExecWait(Proc,_("decompressor"),false) == false)
-        return false;
-   return true;
-}   
-                                                                       /*}}}*/

 // MultiCompress::Child - The writer child                             /*{{{*/
 // ---------------------------------------------------------------------
 /* The child process forks a bunch of compression children and takes 
 // MultiCompress::Child - The writer child                             /*{{{*/
 // ---------------------------------------------------------------------
 /* The child process forks a bunch of compression children and takes 


@@ -371,19 +278,11 @@ bool MultiCompress::CloseOld(int Fd,pid_t Proc)
    is new then the temp files are renamed, otherwise they are erased. */
 bool MultiCompress::Child(int const &FD)
 {
    is new then the temp files are renamed, otherwise they are erased. */
 bool MultiCompress::Child(int const &FD)
 {

-   // Start the compression children.
-   for (Files *I = Outputs; I != 0; I = I->Next)
-   {
-      if (OpenCompress(I->CompressProg,I->CompressProc,I->TmpFile.Fd(),
-                      I->Fd,true) == false)
-        return false;      
-   }
-

    /* Okay, now we just feed data from FD to all the other FDs. Also
       stash a hash of the data to use later. */
    SetNonBlock(FD,false);
    unsigned char Buffer[32*1024];
    /* Okay, now we just feed data from FD to all the other FDs. Also
       stash a hash of the data to use later. */
    SetNonBlock(FD,false);
    unsigned char Buffer[32*1024];

-   unsigned long FileSize = 0;
+   unsigned long long FileSize = 0;

    MD5Summation MD5;
    while (1)
    {
    MD5Summation MD5;
    while (1)
    {


@@ -398,25 +297,14 @@ bool MultiCompress::Child(int const &FD)
       FileSize += Res;
       for (Files *I = Outputs; I != 0; I = I->Next)
       {
       FileSize += Res;
       for (Files *I = Outputs; I != 0; I = I->Next)
       {

-        if (write(I->Fd,Buffer,Res) != Res)
+        if (I->TmpFile.Write(Buffer, Res) == false)

         {
            _error->Errno("write",_("IO to subprocess/file failed"));
            break;
         }
       }      
    }   
         {
            _error->Errno("write",_("IO to subprocess/file failed"));
            break;
         }
       }      
    }   

-   
-   // Close all the writers
-   for (Files *I = Outputs; I != 0; I = I->Next)
-      close(I->Fd);
-   
-   // Wait for the compressors to exit
-   for (Files *I = Outputs; I != 0; I = I->Next)
-   {
-      if (I->CompressProc != -1)
-        ExecWait(I->CompressProc, I->CompressProg.Binary.c_str(), false);
-   }
-   
+

    if (_error->PendingError() == true)
       return false;
    
    if (_error->PendingError() == true)
       return false;
    


@@ -435,31 +323,27 @@ bool MultiCompress::Child(int const &FD)
    // Check the MD5 of the lowest cost entity.
    while (Missing == false)
    {
    // Check the MD5 of the lowest cost entity.
    while (Missing == false)
    {

-      int CompFd = -1;
-      pid_t Proc = -1;
-      if (OpenOld(CompFd,Proc) == false)
+      FileFd CompFd;
+      if (OpenOld(CompFd) == false)

       {
         _error->Discard();
         break;
       }
       {
         _error->Discard();
         break;
       }

-            
+

       // Compute the hash
       MD5Summation OldMD5;
       // Compute the hash
       MD5Summation OldMD5;

-      unsigned long NewFileSize = 0;
+      unsigned long long NewFileSize = 0;

       while (1)
       {
       while (1)
       {

-        int Res = read(CompFd,Buffer,sizeof(Buffer));
+        unsigned long long Res = 0;
+        if (CompFd.Read(Buffer,sizeof(Buffer), &Res) == false)
+           return _error->Errno("read",_("Failed to read while computing MD5"));

         if (Res == 0)
            break;
         if (Res == 0)
            break;

-        if (Res < 0)
-           return _error->Errno("read",_("Failed to read while computing MD5"));

         NewFileSize += Res;
         OldMD5.Add(Buffer,Res);
       }
         NewFileSize += Res;
         OldMD5.Add(Buffer,Res);
       }

-      
-      // Tidy the compressor
-      if (CloseOld(CompFd,Proc) == false)
-        return false;
+      CompFd.Close();

 
       // Check the hash
       if (OldMD5.Result() == MD5.Result() &&
 
       // Check the hash
       if (OldMD5.Result() == MD5.Result() &&


@@ -481,7 +365,7 @@ bool MultiCompress::Child(int const &FD)
    for (Files *I = Outputs; I != 0; I = I->Next)
    {
       // Set the correct file modes
    for (Files *I = Outputs; I != 0; I = I->Next)
    {
       // Set the correct file modes

-      fchmod(I->TmpFile.Fd(),Permissions);
+      chmod(I->TmpFile.Name().c_str(),Permissions);

       
       if (rename(I->TmpFile.Name().c_str(),I->Output.c_str()) != 0)
         _error->Errno("rename",_("Failed to rename %s to %s"),
       
       if (rename(I->TmpFile.Name().c_str(),I->Output.c_str()) != 0)
         _error->Errno("rename",_("Failed to rename %s to %s"),