1 // Copyright (c) 2014 Anthony Towns
3 // This program is free software; you can redistribute it and/or modify
4 // it under the terms of the GNU General Public License as published by
5 // the Free Software Foundation; either version 2 of the License, or
6 // (at your option) any later version.
10 #include <apt-pkg/init.h>
11 #include <apt-pkg/fileutl.h>
12 #include <apt-pkg/error.h>
13 #include <apt-pkg/acquire-method.h>
14 #include <apt-pkg/strutl.h>
15 #include <apt-pkg/hashes.h>
16 #include <apt-pkg/configuration.h>
17 #include "aptmethod.h"
35 #define BLOCK_SIZE (512*1024)
43 explicit MemBlock(size_t size
) : size(size
), next(NULL
)
45 free
= start
= new char[size
];
48 size_t avail(void) { return size
- (free
- start
); }
53 free
= start
= new char[BLOCK_SIZE
];
69 char *add_easy(char *src
, size_t len
, char *last
)
72 for (MemBlock
*k
= this; k
; k
= k
->next
) {
73 if (k
->free
== last
) {
74 if (len
<= k
->avail()) {
75 char *n
= k
->add(src
, len
);
83 } else if (last
>= start
&& last
< free
) {
91 char *add(char *src
, size_t len
) {
94 if (len
> BLOCK_SIZE
) {
95 next
= new MemBlock(len
);
100 return next
->add(src
, len
);
104 memcpy(dst
, src
, len
);
112 * 1. write out <offset> lines unchanged
113 * 2. skip <del_cnt> lines from source
114 * 3. write out <add_cnt> lines (<add>/<add_len>)
118 size_t add_cnt
; /* lines */
119 size_t add_len
; /* bytes */
122 explicit Change(size_t off
)
125 del_cnt
= add_cnt
= add_len
= 0;
129 /* actually, don't write <lines> lines from <add> */
130 void skip_lines(size_t lines
)
133 char *s
= (char*) memchr(add
, '\n', add_len
);
136 add_len
-= (s
- add
);
141 assert(add_cnt
== 0);
152 std::list
<struct Change
> changes
;
153 std::list
<struct Change
>::iterator where
;
154 size_t pos
; // line number is as far left of iterator as possible
156 bool pos_is_okay(void) const
160 std::list
<struct Change
>::const_iterator x
;
161 for (x
= changes
.begin(); x
!= where
; ++x
) {
162 assert(x
!= changes
.end());
163 cpos
+= x
->offset
+ x
->add_cnt
;
173 where
= changes
.end();
177 std::list
<struct Change
>::iterator
begin(void) { return changes
.begin(); }
178 std::list
<struct Change
>::iterator
end(void) { return changes
.end(); }
180 std::list
<struct Change
>::reverse_iterator
rbegin(void) { return changes
.rbegin(); }
181 std::list
<struct Change
>::reverse_iterator
rend(void) { return changes
.rend(); }
183 void add_change(Change c
) {
184 assert(pos_is_okay());
185 go_to_change_for(c
.offset
);
186 assert(pos
+ where
->offset
== c
.offset
);
188 delete_lines(c
.del_cnt
);
189 assert(pos
+ where
->offset
== c
.offset
);
191 assert(pos_is_okay());
192 if (where
->add_len
> 0)
194 assert(where
->add_len
== 0 && where
->add_cnt
== 0);
196 where
->add_len
= c
.add_len
;
197 where
->add_cnt
= c
.add_cnt
;
200 assert(pos_is_okay());
202 assert(pos_is_okay());
208 while (where
->offset
== 0 && where
!= changes
.begin()) {
211 std::list
<struct Change
>::iterator next
= where
;
214 while (next
!= changes
.end() && next
->offset
== 0) {
215 where
->del_cnt
+= next
->del_cnt
;
217 if (next
->add
== NULL
) {
218 next
= changes
.erase(next
);
219 } else if (where
->add
== NULL
) {
220 where
->add
= next
->add
;
221 where
->add_len
= next
->add_len
;
222 where
->add_cnt
= next
->add_cnt
;
223 next
= changes
.erase(next
);
230 void go_to_change_for(size_t line
)
232 while(where
!= changes
.end()) {
237 if (pos
+ where
->offset
+ where
->add_cnt
<= line
) {
241 // line is somewhere in this slot
242 if (line
< pos
+ where
->offset
) {
244 } else if (line
== pos
+ where
->offset
) {
252 /* it goes before this patch */
256 void new_change(void) { insert(where
->offset
); }
258 void insert(size_t offset
)
260 assert(pos_is_okay());
261 assert(where
== changes
.end() || offset
<= where
->offset
);
262 if (where
!= changes
.end())
263 where
->offset
-= offset
;
264 changes
.insert(where
, Change(offset
));
266 assert(pos_is_okay());
269 void split(size_t offset
)
271 assert(pos_is_okay());
273 assert(where
->offset
< offset
);
274 assert(offset
< where
->offset
+ where
->add_cnt
);
276 size_t keep_lines
= offset
- where
->offset
;
278 Change
before(*where
);
282 where
->skip_lines(keep_lines
);
284 before
.add_cnt
= keep_lines
;
285 before
.add_len
-= where
->add_len
;
287 changes
.insert(where
, before
);
289 assert(pos_is_okay());
292 void delete_lines(size_t cnt
)
294 std::list
<struct Change
>::iterator x
= where
;
295 assert(pos_is_okay());
306 if (x
== changes
.end()) {
314 where
->del_cnt
+= del
;
317 assert(pos_is_okay());
321 assert(pos_is_okay());
323 pos
-= where
->offset
+ where
->add_cnt
;
324 assert(pos_is_okay());
328 assert(pos_is_okay());
329 pos
+= where
->offset
+ where
->add_cnt
;
331 assert(pos_is_okay());
336 FileChanges filechanges
;
339 static bool retry_fwrite(char *b
, size_t l
, FileFd
&f
, Hashes
* const start_hash
, Hashes
* const end_hash
= nullptr)
341 if (f
.Write(b
, l
) == false)
344 start_hash
->Add((unsigned char*)b
, l
);
346 end_hash
->Add((unsigned char*)b
, l
);
350 static void dump_rest(FileFd
&o
, FileFd
&i
,
351 Hashes
* const start_hash
, Hashes
* const end_hash
)
353 char buffer
[BLOCK_SIZE
];
354 unsigned long long l
= 0;
355 while (i
.Read(buffer
, sizeof(buffer
), &l
)) {
356 if (l
==0 || !retry_fwrite(buffer
, l
, o
, start_hash
, end_hash
))
361 static void dump_lines(FileFd
&o
, FileFd
&i
, size_t n
,
362 Hashes
* const start_hash
, Hashes
* const end_hash
)
364 char buffer
[BLOCK_SIZE
];
366 if (i
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
368 size_t const l
= strlen(buffer
);
369 if (l
== 0 || buffer
[l
-1] == '\n')
371 retry_fwrite(buffer
, l
, o
, start_hash
, end_hash
);
375 static void skip_lines(FileFd
&i
, int n
, Hashes
* const start_hash
)
377 char buffer
[BLOCK_SIZE
];
379 if (i
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
381 size_t const l
= strlen(buffer
);
382 if (l
== 0 || buffer
[l
-1] == '\n')
385 start_hash
->Add((unsigned char*)buffer
, l
);
389 static void dump_mem(FileFd
&o
, char *p
, size_t s
, Hashes
*hash
) {
390 retry_fwrite(p
, s
, o
, hash
);
395 bool read_diff(FileFd
&f
, Hashes
* const h
)
397 char buffer
[BLOCK_SIZE
];
398 bool cmdwanted
= true;
400 Change
ch(std::numeric_limits
<size_t>::max());
401 if (f
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
402 return _error
->Error("Reading first line of patchfile %s failed", f
.Name().c_str());
410 s
= strtoul(buffer
, &m
, 10);
411 if (unlikely(m
== buffer
|| s
== std::numeric_limits
<unsigned long>::max() || errno
!= 0))
412 return _error
->Error("Parsing patchfile %s failed: Expected an effected line start", f
.Name().c_str());
413 else if (*m
== ',') {
415 e
= strtol(m
, &c
, 10);
416 if (unlikely(m
== c
|| e
== std::numeric_limits
<unsigned long>::max() || errno
!= 0))
417 return _error
->Error("Parsing patchfile %s failed: Expected an effected line end", f
.Name().c_str());
419 return _error
->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f
.Name().c_str(), e
, s
);
425 return _error
->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f
.Name().c_str());
436 if (unlikely(s
== 0))
437 return _error
->Error("Parsing patchfile %s failed: Change command can't effect line zero", f
.Name().c_str());
443 ch
.del_cnt
= e
- s
+ 1;
446 if (unlikely(s
== 0))
447 return _error
->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f
.Name().c_str());
449 ch
.del_cnt
= e
- s
+ 1;
453 filechanges
.add_change(ch
);
456 return _error
->Error("Parsing patchfile %s failed: Unknown command", f
.Name().c_str());
458 } else { /* !cmdwanted */
459 if (strcmp(buffer
, ".\n") == 0) {
461 filechanges
.add_change(ch
);
467 last
= ch
.add
+ ch
.add_len
;
469 add
= add_text
.add_easy(buffer
, l
, last
);
475 filechanges
.add_change(ch
);
478 ch
.offset
+= ch
.add_cnt
;
485 } while(f
.ReadLine(buffer
, sizeof(buffer
)));
489 void write_diff(FileFd
&f
)
491 unsigned long long line
= 0;
492 std::list
<struct Change
>::reverse_iterator ch
;
493 for (ch
= filechanges
.rbegin(); ch
!= filechanges
.rend(); ++ch
) {
494 line
+= ch
->offset
+ ch
->del_cnt
;
497 for (ch
= filechanges
.rbegin(); ch
!= filechanges
.rend(); ++ch
) {
498 std::list
<struct Change
>::reverse_iterator mg_i
, mg_e
= ch
;
499 while (ch
->del_cnt
== 0 && ch
->offset
== 0)
502 if (unlikely(ch
== filechanges
.rend()))
507 if (ch
->add_cnt
> 0) {
508 if (ch
->del_cnt
== 0) {
509 strprintf(buf
, "%llua\n", line
);
510 } else if (ch
->del_cnt
== 1) {
511 strprintf(buf
, "%lluc\n", line
+1);
513 strprintf(buf
, "%llu,%lluc\n", line
+1, line
+ch
->del_cnt
);
515 f
.Write(buf
.c_str(), buf
.length());
519 dump_mem(f
, mg_i
->add
, mg_i
->add_len
, NULL
);
520 } while (mg_i
-- != mg_e
);
523 f
.Write(buf
.c_str(), buf
.length());
524 } else if (ch
->del_cnt
== 1) {
525 strprintf(buf
, "%llud\n", line
+1);
526 f
.Write(buf
.c_str(), buf
.length());
527 } else if (ch
->del_cnt
> 1) {
528 strprintf(buf
, "%llu,%llud\n", line
+1, line
+ch
->del_cnt
);
529 f
.Write(buf
.c_str(), buf
.length());
535 void apply_against_file(FileFd
&out
, FileFd
&in
,
536 Hashes
* const start_hash
= nullptr, Hashes
* const end_hash
= nullptr)
538 std::list
<struct Change
>::iterator ch
;
539 for (ch
= filechanges
.begin(); ch
!= filechanges
.end(); ++ch
) {
540 dump_lines(out
, in
, ch
->offset
, start_hash
, end_hash
);
541 skip_lines(in
, ch
->del_cnt
, start_hash
);
542 dump_mem(out
, ch
->add
, ch
->add_len
, end_hash
);
544 dump_rest(out
, in
, start_hash
, end_hash
);
549 class RredMethod
: public aptMethod
{
554 std::string FileName
;
555 HashStringList ExpectedHashes
;
556 PDiffFile(std::string
const &FileName
, HashStringList
const &ExpectedHashes
) :
557 FileName(FileName
), ExpectedHashes(ExpectedHashes
) {}
560 HashStringList
ReadExpectedHashesForPatch(unsigned int const patch
, std::string
const &Message
)
562 HashStringList ExpectedHashes
;
563 for (char const * const * type
= HashString::SupportedHashes(); *type
!= NULL
; ++type
)
566 strprintf(tagname
, "Patch-%d-%s-Hash", patch
, *type
);
567 std::string
const hashsum
= LookupTag(Message
, tagname
.c_str());
568 if (hashsum
.empty() == false)
569 ExpectedHashes
.push_back(HashString(*type
, hashsum
));
571 return ExpectedHashes
;
575 virtual bool URIAcquire(std::string
const &Message
, FetchItem
*Itm
) APT_OVERRIDE
{
576 Debug
= _config
->FindB("Debug::pkgAcquire::RRed", false);
578 std::string Path
= Get
.Host
+ Get
.Path
; // rred:/path - no host
581 Res
.Filename
= Itm
->DestFile
;
582 if (Itm
->Uri
.empty())
584 Path
= Itm
->DestFile
;
585 Itm
->DestFile
.append(".result");
589 std::vector
<PDiffFile
> patchfiles
;
592 HashStringList StartHashes
;
593 for (char const * const * type
= HashString::SupportedHashes(); *type
!= nullptr; ++type
)
596 strprintf(tagname
, "Start-%s-Hash", *type
);
597 std::string
const hashsum
= LookupTag(Message
, tagname
.c_str());
598 if (hashsum
.empty() == false)
599 StartHashes
.push_back(HashString(*type
, hashsum
));
602 if (FileExists(Path
+ ".ed") == true)
604 HashStringList
const ExpectedHashes
= ReadExpectedHashesForPatch(0, Message
);
605 std::string
const FileName
= Path
+ ".ed";
606 if (ExpectedHashes
.usable() == false)
607 return _error
->Error("No hashes found for uncompressed patch: %s", FileName
.c_str());
608 patchfiles
.push_back(PDiffFile(FileName
, ExpectedHashes
));
612 _error
->PushToStack();
613 std::vector
<std::string
> patches
= GetListOfFilesInDir(flNotFile(Path
), "gz", true, false);
614 _error
->RevertToStack();
616 std::string
const baseName
= Path
+ ".ed.";
617 unsigned int seen_patches
= 0;
618 for (std::vector
<std::string
>::const_iterator p
= patches
.begin();
619 p
!= patches
.end(); ++p
)
621 if (p
->compare(0, baseName
.length(), baseName
) == 0)
623 HashStringList
const ExpectedHashes
= ReadExpectedHashesForPatch(seen_patches
, Message
);
624 if (ExpectedHashes
.usable() == false)
625 return _error
->Error("No hashes found for uncompressed patch %d: %s", seen_patches
, p
->c_str());
626 patchfiles
.push_back(PDiffFile(*p
, ExpectedHashes
));
632 std::string patch_name
;
633 for (std::vector
<PDiffFile
>::iterator I
= patchfiles
.begin();
634 I
!= patchfiles
.end();
637 patch_name
= I
->FileName
;
639 std::clog
<< "Patching " << Path
<< " with " << patch_name
643 Hashes
patch_hash(I
->ExpectedHashes
);
644 // all patches are compressed, even if the name doesn't reflect it
645 if (p
.Open(patch_name
, FileFd::ReadOnly
, FileFd::Gzip
) == false ||
646 patch
.read_diff(p
, &patch_hash
) == false)
648 _error
->DumpErrors(std::cerr
, GlobalError::DEBUG
, false);
652 HashStringList
const hsl
= patch_hash
.GetHashStringList();
653 if (hsl
!= I
->ExpectedHashes
)
654 return _error
->Error("Hash Sum mismatch for uncompressed patch %s", patch_name
.c_str());
658 std::clog
<< "Applying patches against " << Path
659 << " and writing results to " << Itm
->DestFile
663 if (inp
.Open(Path
, FileFd::ReadOnly
, FileFd::Extension
) == false)
665 std::cerr
<< "FAILED to open inp " << Path
<< std::endl
;
666 return _error
->Error("Failed to open inp %s", Path
.c_str());
668 if (out
.Open(Itm
->DestFile
, FileFd::WriteOnly
| FileFd::Create
| FileFd::Empty
| FileFd::BufferedWrite
, FileFd::Extension
) == false)
670 std::cerr
<< "FAILED to open out " << Itm
->DestFile
<< std::endl
;
671 return _error
->Error("Failed to open out %s", Itm
->DestFile
.c_str());
674 Hashes
end_hash(Itm
->ExpectedHashes
);
675 if (StartHashes
.usable())
677 Hashes
start_hash(StartHashes
);
678 patch
.apply_against_file(out
, inp
, &start_hash
, &end_hash
);
679 if (start_hash
.GetHashStringList() != StartHashes
)
680 _error
->Error("The input file hadn't the expected hash!");
683 patch
.apply_against_file(out
, inp
, nullptr, &end_hash
);
688 if (_error
->PendingError() == true) {
689 std::cerr
<< "FAILED to read or write files" << std::endl
;
694 std::clog
<< "rred: finished file patching of " << Path
<< "." << std::endl
;
697 struct stat bufbase
, bufpatch
;
698 if (stat(Path
.c_str(), &bufbase
) != 0 ||
699 stat(patch_name
.c_str(), &bufpatch
) != 0)
700 return _error
->Errno("stat", _("Failed to stat %s"), Path
.c_str());
702 struct timeval times
[2];
703 times
[0].tv_sec
= bufbase
.st_atime
;
704 times
[1].tv_sec
= bufpatch
.st_mtime
;
705 times
[0].tv_usec
= times
[1].tv_usec
= 0;
706 if (utimes(Itm
->DestFile
.c_str(), times
) != 0)
707 return _error
->Errno("utimes",_("Failed to set modification time"));
709 if (stat(Itm
->DestFile
.c_str(), &bufbase
) != 0)
710 return _error
->Errno("stat", _("Failed to stat %s"), Itm
->DestFile
.c_str());
712 Res
.LastModified
= bufbase
.st_mtime
;
713 Res
.Size
= bufbase
.st_size
;
714 Res
.TakeHashes(end_hash
);
721 RredMethod() : aptMethod("rred", "2.0", SendConfig
), Debug(false) {}
724 int main(int argc
, char **argv
)
727 bool just_diff
= true;
736 // Usage: rred -t input output diff ...
737 if (argc
> 1 && strcmp(argv
[1], "-t") == 0) {
738 // Read config files so we see compressors.
739 pkgInitConfig(*_config
);
743 } else if (argc
> 1 && strcmp(argv
[1], "-f") == 0) {
750 for (; i
< argc
; i
++) {
752 if (p
.Open(argv
[i
], FileFd::ReadOnly
) == false) {
753 _error
->DumpErrors(std::cerr
);
756 if (patch
.read_diff(p
, NULL
) == false)
758 _error
->DumpErrors(std::cerr
);
765 std::cerr
<< "Patching " << argv
[2] << " into " << argv
[3] << "\n";
766 inp
.Open(argv
[2], FileFd::ReadOnly
,FileFd::Extension
);
767 out
.Open(argv
[3], FileFd::WriteOnly
| FileFd::Create
| FileFd::Empty
| FileFd::BufferedWrite
, FileFd::Extension
);
768 patch
.apply_against_file(out
, inp
);
770 } else if (just_diff
) {
772 out
.OpenDescriptor(STDOUT_FILENO
, FileFd::WriteOnly
| FileFd::Create
);
773 patch
.write_diff(out
);
777 out
.OpenDescriptor(STDOUT_FILENO
, FileFd::WriteOnly
| FileFd::Create
| FileFd::BufferedWrite
);
778 inp
.OpenDescriptor(STDIN_FILENO
, FileFd::ReadOnly
);
779 patch
.apply_against_file(out
, inp
);