1 // -*- mode: cpp; mode: fold -*-
3 // $Id: multicompress.cc,v 1.4 2003/02/10 07:34:41 doogie Exp $
4 /* ######################################################################
8 This class is very complicated in order to optimize for the common
9 case of its use, writing a large set of compressed files that are
10 different from the old set. It spawns off compressors in parallel
11 to maximize compression throughput and has a separate task managing
12 the data going into the compressors.
14 ##################################################################### */
16 // Include Files /*{{{*/
19 #include <apt-pkg/fileutl.h>
20 #include <apt-pkg/strutl.h>
21 #include <apt-pkg/error.h>
22 #include <apt-pkg/md5.h>
23 #include <apt-pkg/aptconfiguration.h>
24 #include <apt-pkg/hashsum_template.h>
28 #include <sys/types.h>
33 #include "multicompress.h"
40 // MultiCompress::MultiCompress - Constructor /*{{{*/
41 // ---------------------------------------------------------------------
42 /* Setup the file outputs, compression modes and fork the writer child */
43 MultiCompress::MultiCompress(string
const &Output
,string
const &Compress
,
44 mode_t
const &Permissions
,bool const &Write
) :
45 Permissions(Permissions
)
52 /* Parse the compression string, a space separated lists of compresison
54 string::const_iterator I
= Compress
.begin();
55 for (; I
!= Compress
.end();)
57 for (; I
!= Compress
.end() && isspace(*I
); ++I
);
60 string::const_iterator Start
= I
;
61 for (; I
!= Compress
.end() && !isspace(*I
); ++I
);
63 // Find the matching compressor
64 std::vector
<APT::Configuration::Compressor
> Compressors
= APT::Configuration::getCompressors();
65 std::vector
<APT::Configuration::Compressor
>::const_iterator Comp
= Compressors
.begin();
66 for (; Comp
!= Compressors
.end(); ++Comp
)
67 if (stringcmp(Start
,I
,Comp
->Name
.c_str()) == 0)
71 if (Comp
== Compressors
.end())
73 _error
->Warning(_("Unknown compression algorithm '%s'"),string(Start
,I
).c_str());
77 // Create and link in a new output
78 Files
*NewOut
= new Files
;
79 NewOut
->Next
= Outputs
;
81 NewOut
->CompressProg
= *Comp
;
82 NewOut
->Output
= Output
+Comp
->Extension
;
85 if (stat(NewOut
->Output
.c_str(),&St
) == 0)
86 NewOut
->OldMTime
= St
.st_mtime
;
94 /* Open all the temp files now so we can report any errors. File is
95 made unreable to prevent people from touching it during creating. */
96 for (Files
*I
= Outputs
; I
!= 0; I
= I
->Next
)
97 I
->TmpFile
.Open(I
->Output
+ ".new", FileFd::WriteOnly
| FileFd::Create
| FileFd::Empty
, FileFd::Extension
, 0600);
98 if (_error
->PendingError() == true)
103 _error
->Error(_("Compressed output %s needs a compression set"),Output
.c_str());
110 // MultiCompress::~MultiCompress - Destructor /*{{{*/
111 // ---------------------------------------------------------------------
112 /* Just erase the file linked list. */
113 MultiCompress::~MultiCompress()
117 for (; Outputs
!= 0;)
119 Files
*Tmp
= Outputs
->Next
;
125 // MultiCompress::GetStat - Get stat information for compressed files /*{{{*/
126 // ---------------------------------------------------------------------
127 /* This checks each compressed file to make sure it exists and returns
128 stat information for a random file from the collection. False means
129 one or more of the files is missing. */
130 bool MultiCompress::GetStat(string
const &Output
,string
const &Compress
,struct stat
&St
)
132 /* Parse the compression string, a space separated lists of compresison
134 string::const_iterator I
= Compress
.begin();
135 bool DidStat
= false;
136 for (; I
!= Compress
.end();)
138 for (; I
!= Compress
.end() && isspace(*I
); ++I
);
141 string::const_iterator Start
= I
;
142 for (; I
!= Compress
.end() && !isspace(*I
); ++I
);
144 // Find the matching compressor
145 std::vector
<APT::Configuration::Compressor
> Compressors
= APT::Configuration::getCompressors();
146 std::vector
<APT::Configuration::Compressor
>::const_iterator Comp
= Compressors
.begin();
147 for (; Comp
!= Compressors
.end(); ++Comp
)
148 if (stringcmp(Start
,I
,Comp
->Name
.c_str()) == 0)
152 if (Comp
== Compressors
.end())
155 string Name
= Output
+Comp
->Extension
;
156 if (stat(Name
.c_str(),&St
) != 0)
163 // MultiCompress::Start - Start up the writer child /*{{{*/
164 // ---------------------------------------------------------------------
165 /* Fork a child and setup the communication pipe. */
166 bool MultiCompress::Start()
168 // Create a data pipe
169 int Pipe
[2] = {-1,-1};
171 return _error
->Errno("pipe",_("Failed to create IPC pipe to subprocess"));
172 for (int I
= 0; I
!= 2; I
++)
173 SetCloseExec(Pipe
[I
],true);
181 if (_error
->PendingError() == true)
183 _error
->DumpErrors();
190 Input
= fdopen(Pipe
[1],"w");
192 return _error
->Errno("fdopen",_("Failed to create FILE*"));
195 return _error
->Errno("fork",_("Failed to fork"));
199 // MultiCompress::Die - Clean up the writer /*{{{*/
200 // ---------------------------------------------------------------------
202 bool MultiCompress::Die()
209 bool Res
= ExecWait(Outputter
,_("Compress child"),false);
214 // MultiCompress::Finalize - Finish up writing /*{{{*/
215 // ---------------------------------------------------------------------
216 /* This is only necessary for statistics reporting. */
217 bool MultiCompress::Finalize(unsigned long long &OutSize
)
220 if (Input
== 0 || Die() == false)
226 // Check the mtimes to see if the files were replaced.
227 bool Changed
= false;
228 for (Files
*I
= Outputs
; I
!= 0; I
= I
->Next
)
231 if (stat(I
->Output
.c_str(),&St
) != 0)
232 return _error
->Error(_("Internal error, failed to create %s"),
235 if (I
->OldMTime
!= St
.st_mtime
)
239 // Update the mtime if necessary
240 if (UpdateMTime
> 0 &&
241 (Now
- St
.st_mtime
> (signed)UpdateMTime
|| St
.st_mtime
> Now
))
243 utimes(I
->Output
.c_str(), NULL
);
248 // Force the file permissions
249 if (St
.st_mode
!= Permissions
)
250 chmod(I
->Output
.c_str(),Permissions
);
252 OutSize
+= St
.st_size
;
255 if (Changed
== false)
261 // MultiCompress::OpenOld - Open an old file /*{{{*/
262 // ---------------------------------------------------------------------
263 /* This opens one of the original output files, possibly decompressing it. */
264 bool MultiCompress::OpenOld(FileFd
&Fd
)
266 Files
*Best
= Outputs
;
267 for (Files
*I
= Outputs
; I
!= 0; I
= I
->Next
)
268 if (Best
->CompressProg
.Cost
> I
->CompressProg
.Cost
)
272 return Fd
.Open(Best
->Output
, FileFd::ReadOnly
, FileFd::Extension
);
275 // MultiCompress::Child - The writer child /*{{{*/
276 // ---------------------------------------------------------------------
277 /* The child process forks a bunch of compression children and takes
278 input on FD and passes it to all the compressor child. On the way it
279 computes the MD5 of the raw data. After this the raw data in the
280 original files is compared to see if this data is new. If the data
281 is new then the temp files are renamed, otherwise they are erased. */
282 bool MultiCompress::Child(int const &FD
)
284 /* Okay, now we just feed data from FD to all the other FDs. Also
285 stash a hash of the data to use later. */
286 SetNonBlock(FD
,false);
287 unsigned char Buffer
[32*1024];
288 unsigned long long FileSize
= 0;
293 int Res
= read(FD
,Buffer
,sizeof(Buffer
));
301 for (Files
*I
= Outputs
; I
!= 0; I
= I
->Next
)
303 if (I
->TmpFile
.Write(Buffer
, Res
) == false)
305 _error
->Errno("write",_("IO to subprocess/file failed"));
311 if (_error
->PendingError() == true)
314 /* Now we have to copy the files over, or erase them if they
315 have not changed. First find the cheapest decompressor */
316 bool Missing
= false;
317 for (Files
*I
= Outputs
; I
!= 0; I
= I
->Next
)
319 if (I
->OldMTime
== 0)
326 // Check the MD5 of the lowest cost entity.
327 while (Missing
== false)
330 if (OpenOld(CompFd
) == false)
338 unsigned long long NewFileSize
= 0;
341 unsigned long long Res
= 0;
342 if (CompFd
.Read(Buffer
,sizeof(Buffer
), &Res
) == false)
343 return _error
->Errno("read",_("Failed to read while computing MD5"));
347 OldMD5
.Add(Buffer
,Res
);
352 if (OldMD5
.Result() == MD5
.Result() &&
353 FileSize
== NewFileSize
)
355 for (Files
*I
= Outputs
; I
!= 0; I
= I
->Next
)
358 if (unlink(I
->TmpFile
.Name().c_str()) != 0)
359 _error
->Errno("unlink",_("Problem unlinking %s"),
360 I
->TmpFile
.Name().c_str());
362 return !_error
->PendingError();
368 for (Files
*I
= Outputs
; I
!= 0; I
= I
->Next
)
370 // Set the correct file modes
371 chmod(I
->TmpFile
.Name().c_str(),Permissions
);
373 if (rename(I
->TmpFile
.Name().c_str(),I
->Output
.c_str()) != 0)
374 _error
->Errno("rename",_("Failed to rename %s to %s"),
375 I
->TmpFile
.Name().c_str(),I
->Output
.c_str());
379 return !_error
->PendingError();