]>
git.saurik.com Git - apt.git/blob - methods/rred.cc
4 #include <apt-pkg/fileutl.h>
5 #include <apt-pkg/mmap.h>
6 #include <apt-pkg/error.h>
7 #include <apt-pkg/acquire-method.h>
8 #include <apt-pkg/strutl.h>
9 #include <apt-pkg/hashes.h>
10 #include <apt-pkg/configuration.h>
14 #include <sys/types.h>
22 /** \brief RredMethod - ed-style incremential patch method {{{
24 * This method implements a patch functionality similar to "patch --ed" that is
25 * used by the "tiffany" incremental packages download stuff. It differs from
26 * "ed" insofar that it is way more restricted (and therefore secure).
27 * The currently supported ed commands are "<em>c</em>hange", "<em>a</em>dd" and
28 * "<em>d</em>elete" (diff doesn't output any other).
29 * Additionally the records must be reverse sorted by line number and
30 * may not overlap (diff *seems* to produce this kind of output).
32 class RredMethod
: public pkgAcqMethod
{
34 // the size of this doesn't really matter (except for performance)
35 const static int BUF_SIZE
= 1024;
36 // the supported ed commands
37 enum Mode
{MODE_CHANGED
='c', MODE_DELETED
='d', MODE_ADDED
='a'};
39 enum State
{ED_OK
, ED_ORDERING
, ED_PARSER
, ED_FAILURE
, MMAP_FAILED
};
41 State
applyFile(FileFd
&ed_cmds
, FileFd
&in_file
, FileFd
&out_file
,
42 unsigned long &line
, char *buffer
, Hashes
*hash
) const;
43 void ignoreLineInFile(FileFd
&fin
, char *buffer
) const;
44 void copyLinesFromFileToFile(FileFd
&fin
, FileFd
&fout
, unsigned int lines
,
45 Hashes
*hash
, char *buffer
) const;
47 State
patchFile(FileFd
&Patch
, FileFd
&From
, FileFd
&out_file
, Hashes
*hash
) const;
48 State
patchMMap(FileFd
&Patch
, FileFd
&From
, FileFd
&out_file
, Hashes
*hash
) const;
51 // the methods main method
52 virtual bool Fetch(FetchItem
*Itm
);
55 RredMethod() : pkgAcqMethod("1.1",SingleInstance
| SendConfig
), Debug(false) {};
58 /** \brief applyFile - in reverse order with a tail recursion {{{
60 * As it is expected that the commands are in reversed order in the patch file
61 * we check in the first half if the command is valid, but doesn't execute it
62 * and move a step deeper. After reaching the end of the file we apply the
63 * patches in the correct order: last found command first.
65 * \param ed_cmds patch file to apply
66 * \param in_file base file we want to patch
67 * \param out_file file to write the patched result to
68 * \param line of command operation
69 * \param buffer internal used read/write buffer
70 * \param hash the created file for correctness
71 * \return the success State of the ed command executor
73 RredMethod::State
RredMethod::applyFile(FileFd
&ed_cmds
, FileFd
&in_file
, FileFd
&out_file
,
74 unsigned long &line
, char *buffer
, Hashes
*hash
) const {
75 // get the current command and parse it
76 if (ed_cmds
.ReadLine(buffer
, BUF_SIZE
) == NULL
) {
78 std::clog
<< "rred: encounter end of file - we can start patching now." << std::endl
;
83 // parse in the effected linenumbers
86 unsigned long const startline
= strtol(buffer
, &idx
, 10);
87 if (errno
== ERANGE
|| errno
== EINVAL
) {
88 _error
->Errno("rred", "startline is an invalid number");
91 if (startline
> line
) {
92 _error
->Error("rred: The start line (%lu) of the next command is higher than the last line (%lu). This is not allowed.", startline
, line
);
95 unsigned long stopline
;
99 stopline
= strtol(idx
, &idx
, 10);
100 if (errno
== ERANGE
|| errno
== EINVAL
) {
101 _error
->Errno("rred", "stopline is an invalid number");
106 stopline
= startline
;
110 // which command to execute on this line(s)?
114 std::clog
<< "Change from line " << startline
<< " to " << stopline
<< std::endl
;
118 std::clog
<< "Insert after line " << startline
<< std::endl
;
122 std::clog
<< "Delete from line " << startline
<< " to " << stopline
<< std::endl
;
125 _error
->Error("rred: Unknown ed command '%c'. Abort.", *idx
);
128 unsigned char mode
= *idx
;
130 // save the current position
131 unsigned const long long pos
= ed_cmds
.Tell();
133 // if this is add or change then go to the next full stop
134 unsigned int data_length
= 0;
135 if (mode
== MODE_CHANGED
|| mode
== MODE_ADDED
) {
137 ignoreLineInFile(ed_cmds
, buffer
);
140 while (strncmp(buffer
, ".", 1) != 0);
141 data_length
--; // the dot should not be copied
144 // do the recursive call - the last command is the one we need to execute at first
145 const State child
= applyFile(ed_cmds
, in_file
, out_file
, line
, buffer
, hash
);
146 if (child
!= ED_OK
) {
150 // change and delete are working on "line" - add is done after "line"
151 if (mode
!= MODE_ADDED
)
154 // first wind to the current position and copy over all unchanged lines
155 if (line
< startline
) {
156 copyLinesFromFileToFile(in_file
, out_file
, (startline
- line
), hash
, buffer
);
160 if (mode
!= MODE_ADDED
)
163 // include data from ed script
164 if (mode
== MODE_CHANGED
|| mode
== MODE_ADDED
) {
166 copyLinesFromFileToFile(ed_cmds
, out_file
, data_length
, hash
, buffer
);
169 // ignore the corresponding number of lines from input
170 if (mode
== MODE_CHANGED
|| mode
== MODE_DELETED
) {
171 while (line
< stopline
) {
172 ignoreLineInFile(in_file
, buffer
);
179 void RredMethod::copyLinesFromFileToFile(FileFd
&fin
, FileFd
&fout
, unsigned int lines
,/*{{{*/
180 Hashes
*hash
, char *buffer
) const {
181 while (0 < lines
--) {
183 fin
.ReadLine(buffer
, BUF_SIZE
);
184 unsigned long long const towrite
= strlen(buffer
);
185 fout
.Write(buffer
, towrite
);
186 hash
->Add((unsigned char*)buffer
, towrite
);
187 } while (strlen(buffer
) == (BUF_SIZE
- 1) &&
188 buffer
[BUF_SIZE
- 2] != '\n');
192 void RredMethod::ignoreLineInFile(FileFd
&fin
, char *buffer
) const { /*{{{*/
193 fin
.ReadLine(buffer
, BUF_SIZE
);
194 while (strlen(buffer
) == (BUF_SIZE
- 1) &&
195 buffer
[BUF_SIZE
- 2] != '\n') {
196 fin
.ReadLine(buffer
, BUF_SIZE
);
201 RredMethod::State
RredMethod::patchFile(FileFd
&Patch
, FileFd
&From
, /*{{{*/
202 FileFd
&out_file
, Hashes
*hash
) const {
203 char buffer
[BUF_SIZE
];
205 /* we do a tail recursion to read the commands in the right order */
206 unsigned long line
= -1; // assign highest possible value
207 State
const result
= applyFile(Patch
, From
, out_file
, line
, buffer
, hash
);
209 /* read the rest from infile */
210 if (result
== ED_OK
) {
211 while (From
.ReadLine(buffer
, BUF_SIZE
) != NULL
) {
212 unsigned long long const towrite
= strlen(buffer
);
213 out_file
.Write(buffer
, towrite
);
214 hash
->Add((unsigned char*)buffer
, towrite
);
220 /* struct EdCommand {{{*/
221 #ifdef _POSIX_MAPPED_FILES
230 #define IOV_COUNT 1024 /* Don't really want IOV_MAX since it can be arbitrarily large */
231 static ssize_t
retry_writev(int fd
, const struct iovec
*iov
, int iovcnt
) {
236 Res
= writev(fd
, iov
+ i
, iovcnt
);
237 if (Res
< 0 && errno
== EINTR
)
240 return _error
->Errno("writev",_("Write error"));
243 } while (Res
> 0 && iovcnt
> 0);
248 RredMethod::State
RredMethod::patchMMap(FileFd
&Patch
, FileFd
&From
, /*{{{*/
249 FileFd
&out_file
, Hashes
*hash
) const {
250 #ifdef _POSIX_MAPPED_FILES
251 MMap
ed_cmds(Patch
, MMap::ReadOnly
);
252 MMap
in_file(From
, MMap::ReadOnly
);
254 unsigned long long const ed_size
= ed_cmds
.Size();
255 unsigned long long const in_size
= in_file
.Size();
256 if (ed_size
== 0 || in_size
== 0)
259 EdCommand
* commands
= 0;
260 size_t command_count
= 0;
261 size_t command_alloc
= 0;
263 const char* begin
= (char*) ed_cmds
.Data();
264 const char* end
= begin
;
265 const char* ed_end
= (char*) ed_cmds
.Data() + ed_size
;
267 const char* input
= (char*) in_file
.Data();
268 const char* input_end
= (char*) in_file
.Data() + in_size
;
272 /* 1. Parse entire script. It is executed in reverse order, so we cather it
273 * in the `commands' buffer first
281 while(begin
!= ed_end
&& *begin
== '\n')
283 while(end
!= ed_end
&& *end
!= '\n')
285 if(end
== ed_end
&& begin
== end
)
288 /* Determine command range */
289 const char* tmp
= begin
;
292 /* atoll is safe despite lacking NUL-termination; we know there's an
293 * alphabetic character at end[-1]
296 cmd
.first_line
= atol(begin
);
297 cmd
.last_line
= cmd
.first_line
;
301 cmd
.first_line
= atol(begin
);
302 cmd
.last_line
= atol(tmp
+ 1);
308 // which command to execute on this line(s)?
312 std::clog
<< "Change from line " << cmd
.first_line
<< " to " << cmd
.last_line
<< std::endl
;
316 std::clog
<< "Insert after line " << cmd
.first_line
<< std::endl
;
320 std::clog
<< "Delete from line " << cmd
.first_line
<< " to " << cmd
.last_line
<< std::endl
;
323 _error
->Error("rred: Unknown ed command '%c'. Abort.", end
[-1]);
329 /* Determine the size of the inserted text, so we don't have to scan this
336 if(cmd
.type
== MODE_ADDED
|| cmd
.type
== MODE_CHANGED
) {
337 cmd
.data_start
= begin
- (char*) ed_cmds
.Data();
338 while(end
!= ed_end
) {
340 if(end
[-1] == '.' && end
[-2] == '\n')
346 cmd
.data_end
= end
- (char*) ed_cmds
.Data() - 1;
350 if(command_count
== command_alloc
) {
351 command_alloc
= (command_alloc
+ 64) * 3 / 2;
352 EdCommand
* newCommands
= (EdCommand
*) realloc(commands
, command_alloc
* sizeof(EdCommand
));
353 if (newCommands
== NULL
) {
357 commands
= newCommands
;
359 commands
[command_count
++] = cmd
;
362 struct iovec
* iov
= new struct iovec
[IOV_COUNT
];
365 size_t amount
, remaining
;
369 /* 2. Execute script. We gather writes in a `struct iov' array, and flush
370 * using writev to minimize the number of system calls. Data is read
371 * directly from the memory mappings of the input file and the script.
374 for(i
= command_count
; i
-- > 0; ) {
376 if(cmd
->type
== MODE_ADDED
)
377 amount
= cmd
->first_line
+ 1;
379 amount
= cmd
->first_line
;
383 while(line
!= amount
) {
384 input
= (const char*) memchr(input
, '\n', input_end
- input
);
391 iov
[iov_size
].iov_base
= (void*) begin
;
392 iov
[iov_size
].iov_len
= input
- begin
;
393 hash
->Add((const unsigned char*) begin
, input
- begin
);
395 if(++iov_size
== IOV_COUNT
) {
396 retry_writev(out_file
.Fd(), iov
, IOV_COUNT
);
401 if(cmd
->type
== MODE_DELETED
|| cmd
->type
== MODE_CHANGED
) {
402 remaining
= (cmd
->last_line
- cmd
->first_line
) + 1;
405 input
= (const char*) memchr(input
, '\n', input_end
- input
);
413 if(cmd
->type
== MODE_CHANGED
|| cmd
->type
== MODE_ADDED
) {
414 if(cmd
->data_end
!= cmd
->data_start
) {
415 iov
[iov_size
].iov_base
= (void*) ((char*)ed_cmds
.Data() + cmd
->data_start
);
416 iov
[iov_size
].iov_len
= cmd
->data_end
- cmd
->data_start
;
417 hash
->Add((const unsigned char*) ((char*)ed_cmds
.Data() + cmd
->data_start
),
418 iov
[iov_size
].iov_len
);
420 if(++iov_size
== IOV_COUNT
) {
421 retry_writev(out_file
.Fd(), iov
, IOV_COUNT
);
428 if(input
!= input_end
) {
429 iov
[iov_size
].iov_base
= (void*) input
;
430 iov
[iov_size
].iov_len
= input_end
- input
;
431 hash
->Add((const unsigned char*) input
, input_end
- input
);
436 retry_writev(out_file
.Fd(), iov
, iov_size
);
440 for(i
= 0; i
< iov_size
; i
+= IOV_COUNT
) {
441 if(iov_size
- i
< IOV_COUNT
)
442 retry_writev(out_file
.Fd(), iov
+ i
, iov_size
- i
);
444 retry_writev(out_file
.Fd(), iov
+ i
, IOV_COUNT
);
456 bool RredMethod::Fetch(FetchItem
*Itm
) /*{{{*/
458 Debug
= _config
->FindB("Debug::pkgAcquire::RRed", false);
460 std::string Path
= Get
.Host
+ Get
.Path
; // To account for relative paths
463 Res
.Filename
= Itm
->DestFile
;
464 if (Itm
->Uri
.empty() == true) {
465 Path
= Itm
->DestFile
;
466 Itm
->DestFile
.append(".result");
470 std::string lastPatchName
;
473 // check for a single ed file
474 if (FileExists(Path
+".ed") == true)
477 std::clog
<< "Patching " << Path
<< " with " << Path
478 << ".ed and putting result into " << Itm
->DestFile
<< std::endl
;
480 // Open the source and destination files
481 lastPatchName
= Path
+ ".ed";
482 FileFd
From(Path
,FileFd::ReadOnly
);
483 FileFd
To(Itm
->DestFile
,FileFd::WriteAtomic
);
485 FileFd
Patch(lastPatchName
, FileFd::ReadOnly
, FileFd::Gzip
);
486 if (_error
->PendingError() == true)
489 // now do the actual patching
490 State
const result
= patchMMap(Patch
, From
, To
, &Hash
);
491 if (result
== MMAP_FAILED
) {
492 // retry with patchFile
495 To
.Open(Itm
->DestFile
,FileFd::WriteAtomic
);
496 if (_error
->PendingError() == true)
498 if (patchFile(Patch
, From
, To
, &Hash
) != ED_OK
) {
499 return _error
->WarningE("rred", _("Could not patch %s with mmap and with file operation usage - the patch seems to be corrupt."), Path
.c_str());
500 } else if (Debug
== true) {
501 std::clog
<< "rred: finished file patching of " << Path
<< " after mmap failed." << std::endl
;
503 } else if (result
!= ED_OK
) {
504 return _error
->Errno("rred", _("Could not patch %s with mmap (but no mmap specific fail) - the patch seems to be corrupt."), Path
.c_str());
505 } else if (Debug
== true) {
506 std::clog
<< "rred: finished mmap patching of " << Path
<< std::endl
;
509 // write out the result
517 std::clog
<< "Patching " << Path
<< " with all " << Path
<< ".ed.*.gz files and "
518 << "putting result into " << Itm
->DestFile
<< std::endl
;
520 int From
= open(Path
.c_str(), O_RDONLY
);
521 unlink(Itm
->DestFile
.c_str());
522 int To
= open(Itm
->DestFile
.c_str(), O_WRONLY
| O_CREAT
| O_EXCL
, 0644);
523 SetCloseExec(From
, false);
524 SetCloseExec(To
, false);
526 _error
->PushToStack();
527 std::vector
<std::string
> patches
= GetListOfFilesInDir(flNotFile(Path
), "gz", true, false);
528 _error
->RevertToStack();
530 std::string externalrred
= _config
->Find("Dir::Bin::rred", "/usr/bin/diffindex-rred");
531 std::vector
<const char *> Args
;
533 Args
.push_back(externalrred
.c_str());
535 std::string
const baseName
= Path
+ ".ed.";
536 for (std::vector
<std::string
>::const_iterator p
= patches
.begin();
537 p
!= patches
.end(); ++p
)
538 if (p
->compare(0, baseName
.length(), baseName
) == 0)
539 Args
.push_back(p
->c_str());
541 Args
.push_back(NULL
);
543 pid_t Patcher
= ExecFork();
545 dup2(From
, STDIN_FILENO
);
546 dup2(To
, STDOUT_FILENO
);
548 execvp(Args
[0], (char **) &Args
[0]);
549 std::cerr
<< "Failed to execute patcher " << Args
[0] << "!" << std::endl
;
552 // last is NULL, so the one before is the last patch
553 lastPatchName
= Args
[Args
.size() - 2];
555 if (ExecWait(Patcher
, "rred") == false)
556 return _error
->Errno("rred", "Patching via external rred failed");
562 if (stat(Itm
->DestFile
.c_str(), &Buf
) != 0)
563 return _error
->Errno("stat",_("Failed to stat"));
565 To
= open(Path
.c_str(), O_RDONLY
);
566 Hash
.AddFD(To
, Buf
.st_size
);
570 /* Transfer the modification times from the patch file
571 to be able to see in which state the file should be
572 and use the access time from the "old" file */
573 struct stat BufBase
, BufPatch
;
574 if (stat(Path
.c_str(),&BufBase
) != 0 ||
575 stat(lastPatchName
.c_str(), &BufPatch
) != 0)
576 return _error
->Errno("stat",_("Failed to stat"));
578 struct utimbuf TimeBuf
;
579 TimeBuf
.actime
= BufBase
.st_atime
;
580 TimeBuf
.modtime
= BufPatch
.st_mtime
;
581 if (utime(Itm
->DestFile
.c_str(),&TimeBuf
) != 0)
582 return _error
->Errno("utime",_("Failed to set modification time"));
584 if (stat(Itm
->DestFile
.c_str(),&BufBase
) != 0)
585 return _error
->Errno("stat",_("Failed to stat"));
588 Res
.LastModified
= BufBase
.st_mtime
;
589 Res
.Size
= BufBase
.st_size
;
590 Res
.TakeHashes(Hash
);
596 /** \brief Wrapper class for testing rred */ /*{{{*/
597 class TestRredMethod
: public RredMethod
{
599 /** \brief Run rred in debug test mode
601 * This method can be used to run the rred method outside
602 * of the "normal" acquire environment for easier testing.
604 * \param base basename of all files involved in this rred test
606 bool Run(char const *base
) {
607 _config
->CndSet("Debug::pkgAcquire::RRed", "true");
608 FetchItem
*test
= new FetchItem
;
609 test
->DestFile
= base
;
614 /** \brief Starter for the rred method (or its test method) {{{
616 * Used without parameters is the normal behavior for methods for
617 * the APT acquire system. While this works great for the acquire system
618 * it is very hard to test the method and therefore the method also
619 * accepts one parameter which will switch it directly to debug test mode:
620 * The test mode expects that if "Testfile" is given as parameter
621 * the file "Testfile" should be ed-style patched with "Testfile.ed"
622 * and will write the result to "Testfile.result".
624 int main(int argc
, char *argv
[]) {
630 bool result
= Mth
.Run(argv
[1]);
631 _error
->DumpErrors();