1 // Copyright (c) 2014 Anthony Towns
3 // This program is free software; you can redistribute it and/or modify
4 // it under the terms of the GNU General Public License as published by
5 // the Free Software Foundation; either version 2 of the License, or
6 // (at your option) any later version.
10 #include <apt-pkg/init.h>
11 #include <apt-pkg/fileutl.h>
12 #include <apt-pkg/error.h>
13 #include <apt-pkg/strutl.h>
14 #include <apt-pkg/hashes.h>
15 #include <apt-pkg/configuration.h>
16 #include "aptmethod.h"
34 #define BLOCK_SIZE (512*1024)
42 explicit MemBlock(size_t size
) : size(size
), next(NULL
)
44 free
= start
= new char[size
];
47 size_t avail(void) { return size
- (free
- start
); }
52 free
= start
= new char[BLOCK_SIZE
];
68 char *add_easy(char *src
, size_t len
, char *last
)
71 for (MemBlock
*k
= this; k
; k
= k
->next
) {
72 if (k
->free
== last
) {
73 if (len
<= k
->avail()) {
74 char *n
= k
->add(src
, len
);
82 } else if (last
>= start
&& last
< free
) {
90 char *add(char *src
, size_t len
) {
93 if (len
> BLOCK_SIZE
) {
94 next
= new MemBlock(len
);
99 return next
->add(src
, len
);
103 memcpy(dst
, src
, len
);
111 * 1. write out <offset> lines unchanged
112 * 2. skip <del_cnt> lines from source
113 * 3. write out <add_cnt> lines (<add>/<add_len>)
117 size_t add_cnt
; /* lines */
118 size_t add_len
; /* bytes */
121 explicit Change(size_t off
)
124 del_cnt
= add_cnt
= add_len
= 0;
128 /* actually, don't write <lines> lines from <add> */
129 void skip_lines(size_t lines
)
132 char *s
= (char*) memchr(add
, '\n', add_len
);
135 add_len
-= (s
- add
);
140 assert(add_cnt
== 0);
151 std::list
<struct Change
> changes
;
152 std::list
<struct Change
>::iterator where
;
153 size_t pos
; // line number is as far left of iterator as possible
155 bool pos_is_okay(void) const
159 std::list
<struct Change
>::const_iterator x
;
160 for (x
= changes
.begin(); x
!= where
; ++x
) {
161 assert(x
!= changes
.end());
162 cpos
+= x
->offset
+ x
->add_cnt
;
172 where
= changes
.end();
176 std::list
<struct Change
>::iterator
begin(void) { return changes
.begin(); }
177 std::list
<struct Change
>::iterator
end(void) { return changes
.end(); }
179 std::list
<struct Change
>::reverse_iterator
rbegin(void) { return changes
.rbegin(); }
180 std::list
<struct Change
>::reverse_iterator
rend(void) { return changes
.rend(); }
182 void add_change(Change c
) {
183 assert(pos_is_okay());
184 go_to_change_for(c
.offset
);
185 assert(pos
+ where
->offset
== c
.offset
);
187 delete_lines(c
.del_cnt
);
188 assert(pos
+ where
->offset
== c
.offset
);
190 assert(pos_is_okay());
191 if (where
->add_len
> 0)
193 assert(where
->add_len
== 0 && where
->add_cnt
== 0);
195 where
->add_len
= c
.add_len
;
196 where
->add_cnt
= c
.add_cnt
;
199 assert(pos_is_okay());
201 assert(pos_is_okay());
207 while (where
->offset
== 0 && where
!= changes
.begin()) {
210 std::list
<struct Change
>::iterator next
= where
;
213 while (next
!= changes
.end() && next
->offset
== 0) {
214 where
->del_cnt
+= next
->del_cnt
;
216 if (next
->add
== NULL
) {
217 next
= changes
.erase(next
);
218 } else if (where
->add
== NULL
) {
219 where
->add
= next
->add
;
220 where
->add_len
= next
->add_len
;
221 where
->add_cnt
= next
->add_cnt
;
222 next
= changes
.erase(next
);
229 void go_to_change_for(size_t line
)
231 while(where
!= changes
.end()) {
236 if (pos
+ where
->offset
+ where
->add_cnt
<= line
) {
240 // line is somewhere in this slot
241 if (line
< pos
+ where
->offset
) {
243 } else if (line
== pos
+ where
->offset
) {
251 /* it goes before this patch */
255 void new_change(void) { insert(where
->offset
); }
257 void insert(size_t offset
)
259 assert(pos_is_okay());
260 assert(where
== changes
.end() || offset
<= where
->offset
);
261 if (where
!= changes
.end())
262 where
->offset
-= offset
;
263 changes
.insert(where
, Change(offset
));
265 assert(pos_is_okay());
268 void split(size_t offset
)
270 assert(pos_is_okay());
272 assert(where
->offset
< offset
);
273 assert(offset
< where
->offset
+ where
->add_cnt
);
275 size_t keep_lines
= offset
- where
->offset
;
277 Change
before(*where
);
281 where
->skip_lines(keep_lines
);
283 before
.add_cnt
= keep_lines
;
284 before
.add_len
-= where
->add_len
;
286 changes
.insert(where
, before
);
288 assert(pos_is_okay());
291 void delete_lines(size_t cnt
)
293 std::list
<struct Change
>::iterator x
= where
;
294 assert(pos_is_okay());
305 if (x
== changes
.end()) {
313 where
->del_cnt
+= del
;
316 assert(pos_is_okay());
320 assert(pos_is_okay());
322 pos
-= where
->offset
+ where
->add_cnt
;
323 assert(pos_is_okay());
327 assert(pos_is_okay());
328 pos
+= where
->offset
+ where
->add_cnt
;
330 assert(pos_is_okay());
335 FileChanges filechanges
;
338 static bool retry_fwrite(char *b
, size_t l
, FileFd
&f
, Hashes
* const start_hash
, Hashes
* const end_hash
= nullptr) APT_NONNULL(1)
340 if (f
.Write(b
, l
) == false)
343 start_hash
->Add((unsigned char*)b
, l
);
345 end_hash
->Add((unsigned char*)b
, l
);
349 static void dump_rest(FileFd
&o
, FileFd
&i
,
350 Hashes
* const start_hash
, Hashes
* const end_hash
)
352 char buffer
[BLOCK_SIZE
];
353 unsigned long long l
= 0;
354 while (i
.Read(buffer
, sizeof(buffer
), &l
)) {
355 if (l
==0 || !retry_fwrite(buffer
, l
, o
, start_hash
, end_hash
))
360 static void dump_lines(FileFd
&o
, FileFd
&i
, size_t n
,
361 Hashes
* const start_hash
, Hashes
* const end_hash
)
363 char buffer
[BLOCK_SIZE
];
365 if (i
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
367 size_t const l
= strlen(buffer
);
368 if (l
== 0 || buffer
[l
-1] == '\n')
370 retry_fwrite(buffer
, l
, o
, start_hash
, end_hash
);
374 static void skip_lines(FileFd
&i
, int n
, Hashes
* const start_hash
)
376 char buffer
[BLOCK_SIZE
];
378 if (i
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
380 size_t const l
= strlen(buffer
);
381 if (l
== 0 || buffer
[l
-1] == '\n')
384 start_hash
->Add((unsigned char*)buffer
, l
);
388 static void dump_mem(FileFd
&o
, char *p
, size_t s
, Hashes
*hash
) APT_NONNULL(2) {
389 retry_fwrite(p
, s
, o
, nullptr, hash
);
394 bool read_diff(FileFd
&f
, Hashes
* const h
)
396 char buffer
[BLOCK_SIZE
];
397 bool cmdwanted
= true;
399 Change
ch(std::numeric_limits
<size_t>::max());
400 if (f
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
401 return _error
->Error("Reading first line of patchfile %s failed", f
.Name().c_str());
409 s
= strtoul(buffer
, &m
, 10);
410 if (unlikely(m
== buffer
|| s
== std::numeric_limits
<unsigned long>::max() || errno
!= 0))
411 return _error
->Error("Parsing patchfile %s failed: Expected an effected line start", f
.Name().c_str());
412 else if (*m
== ',') {
414 e
= strtol(m
, &c
, 10);
415 if (unlikely(m
== c
|| e
== std::numeric_limits
<unsigned long>::max() || errno
!= 0))
416 return _error
->Error("Parsing patchfile %s failed: Expected an effected line end", f
.Name().c_str());
418 return _error
->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f
.Name().c_str(), e
, s
);
424 return _error
->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f
.Name().c_str());
435 if (unlikely(s
== 0))
436 return _error
->Error("Parsing patchfile %s failed: Change command can't effect line zero", f
.Name().c_str());
442 ch
.del_cnt
= e
- s
+ 1;
445 if (unlikely(s
== 0))
446 return _error
->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f
.Name().c_str());
448 ch
.del_cnt
= e
- s
+ 1;
452 filechanges
.add_change(ch
);
455 return _error
->Error("Parsing patchfile %s failed: Unknown command", f
.Name().c_str());
457 } else { /* !cmdwanted */
458 if (strcmp(buffer
, ".\n") == 0) {
460 filechanges
.add_change(ch
);
466 last
= ch
.add
+ ch
.add_len
;
468 add
= add_text
.add_easy(buffer
, l
, last
);
474 filechanges
.add_change(ch
);
477 ch
.offset
+= ch
.add_cnt
;
484 } while(f
.ReadLine(buffer
, sizeof(buffer
)));
488 void write_diff(FileFd
&f
)
490 unsigned long long line
= 0;
491 std::list
<struct Change
>::reverse_iterator ch
;
492 for (ch
= filechanges
.rbegin(); ch
!= filechanges
.rend(); ++ch
) {
493 line
+= ch
->offset
+ ch
->del_cnt
;
496 for (ch
= filechanges
.rbegin(); ch
!= filechanges
.rend(); ++ch
) {
497 std::list
<struct Change
>::reverse_iterator mg_i
, mg_e
= ch
;
498 while (ch
->del_cnt
== 0 && ch
->offset
== 0)
501 if (unlikely(ch
== filechanges
.rend()))
506 if (ch
->add_cnt
> 0) {
507 if (ch
->del_cnt
== 0) {
508 strprintf(buf
, "%llua\n", line
);
509 } else if (ch
->del_cnt
== 1) {
510 strprintf(buf
, "%lluc\n", line
+1);
512 strprintf(buf
, "%llu,%lluc\n", line
+1, line
+ch
->del_cnt
);
514 f
.Write(buf
.c_str(), buf
.length());
518 dump_mem(f
, mg_i
->add
, mg_i
->add_len
, NULL
);
519 } while (mg_i
-- != mg_e
);
522 f
.Write(buf
.c_str(), buf
.length());
523 } else if (ch
->del_cnt
== 1) {
524 strprintf(buf
, "%llud\n", line
+1);
525 f
.Write(buf
.c_str(), buf
.length());
526 } else if (ch
->del_cnt
> 1) {
527 strprintf(buf
, "%llu,%llud\n", line
+1, line
+ch
->del_cnt
);
528 f
.Write(buf
.c_str(), buf
.length());
534 void apply_against_file(FileFd
&out
, FileFd
&in
,
535 Hashes
* const start_hash
= nullptr, Hashes
* const end_hash
= nullptr)
537 std::list
<struct Change
>::iterator ch
;
538 for (ch
= filechanges
.begin(); ch
!= filechanges
.end(); ++ch
) {
539 dump_lines(out
, in
, ch
->offset
, start_hash
, end_hash
);
540 skip_lines(in
, ch
->del_cnt
, start_hash
);
541 if (ch
->add_len
!= 0)
542 dump_mem(out
, ch
->add
, ch
->add_len
, end_hash
);
544 dump_rest(out
, in
, start_hash
, end_hash
);
549 class RredMethod
: public aptMethod
{
554 std::string FileName
;
555 HashStringList ExpectedHashes
;
556 PDiffFile(std::string
const &FileName
, HashStringList
const &ExpectedHashes
) :
557 FileName(FileName
), ExpectedHashes(ExpectedHashes
) {}
560 HashStringList
ReadExpectedHashesForPatch(unsigned int const patch
, std::string
const &Message
)
562 HashStringList ExpectedHashes
;
563 for (char const * const * type
= HashString::SupportedHashes(); *type
!= NULL
; ++type
)
566 strprintf(tagname
, "Patch-%d-%s-Hash", patch
, *type
);
567 std::string
const hashsum
= LookupTag(Message
, tagname
.c_str());
568 if (hashsum
.empty() == false)
569 ExpectedHashes
.push_back(HashString(*type
, hashsum
));
571 return ExpectedHashes
;
575 virtual bool URIAcquire(std::string
const &Message
, FetchItem
*Itm
) APT_OVERRIDE
{
576 Debug
= DebugEnabled();
578 std::string Path
= Get
.Host
+ Get
.Path
; // rred:/path - no host
581 Res
.Filename
= Itm
->DestFile
;
582 if (Itm
->Uri
.empty())
584 Path
= Itm
->DestFile
;
585 Itm
->DestFile
.append(".result");
589 std::vector
<PDiffFile
> patchfiles
;
592 HashStringList StartHashes
;
593 for (char const * const * type
= HashString::SupportedHashes(); *type
!= nullptr; ++type
)
596 strprintf(tagname
, "Start-%s-Hash", *type
);
597 std::string
const hashsum
= LookupTag(Message
, tagname
.c_str());
598 if (hashsum
.empty() == false)
599 StartHashes
.push_back(HashString(*type
, hashsum
));
602 if (FileExists(Path
+ ".ed") == true)
604 HashStringList
const ExpectedHashes
= ReadExpectedHashesForPatch(0, Message
);
605 std::string
const FileName
= Path
+ ".ed";
606 if (ExpectedHashes
.usable() == false)
607 return _error
->Error("No hashes found for uncompressed patch: %s", FileName
.c_str());
608 patchfiles
.push_back(PDiffFile(FileName
, ExpectedHashes
));
612 _error
->PushToStack();
613 std::vector
<std::string
> patches
= GetListOfFilesInDir(flNotFile(Path
), "gz", true, false);
614 _error
->RevertToStack();
616 std::string
const baseName
= Path
+ ".ed.";
617 unsigned int seen_patches
= 0;
618 for (std::vector
<std::string
>::const_iterator p
= patches
.begin();
619 p
!= patches
.end(); ++p
)
621 if (p
->compare(0, baseName
.length(), baseName
) == 0)
623 HashStringList
const ExpectedHashes
= ReadExpectedHashesForPatch(seen_patches
, Message
);
624 if (ExpectedHashes
.usable() == false)
625 return _error
->Error("No hashes found for uncompressed patch %d: %s", seen_patches
, p
->c_str());
626 patchfiles
.push_back(PDiffFile(*p
, ExpectedHashes
));
632 std::string patch_name
;
633 for (std::vector
<PDiffFile
>::iterator I
= patchfiles
.begin();
634 I
!= patchfiles
.end();
637 patch_name
= I
->FileName
;
639 std::clog
<< "Patching " << Path
<< " with " << patch_name
643 Hashes
patch_hash(I
->ExpectedHashes
);
644 // all patches are compressed, even if the name doesn't reflect it
645 if (p
.Open(patch_name
, FileFd::ReadOnly
, FileFd::Gzip
) == false ||
646 patch
.read_diff(p
, &patch_hash
) == false)
648 _error
->DumpErrors(std::cerr
, GlobalError::DEBUG
, false);
652 HashStringList
const hsl
= patch_hash
.GetHashStringList();
653 if (hsl
!= I
->ExpectedHashes
)
654 return _error
->Error("Hash Sum mismatch for uncompressed patch %s", patch_name
.c_str());
658 std::clog
<< "Applying patches against " << Path
659 << " and writing results to " << Itm
->DestFile
663 if (inp
.Open(Path
, FileFd::ReadOnly
, FileFd::Extension
) == false)
665 std::cerr
<< "FAILED to open inp " << Path
<< std::endl
;
666 return _error
->Error("Failed to open inp %s", Path
.c_str());
668 if (out
.Open(Itm
->DestFile
, FileFd::WriteOnly
| FileFd::Create
| FileFd::Empty
| FileFd::BufferedWrite
, FileFd::Extension
) == false)
670 std::cerr
<< "FAILED to open out " << Itm
->DestFile
<< std::endl
;
671 return _error
->Error("Failed to open out %s", Itm
->DestFile
.c_str());
674 Hashes
end_hash(Itm
->ExpectedHashes
);
675 if (StartHashes
.usable())
677 Hashes
start_hash(StartHashes
);
678 patch
.apply_against_file(out
, inp
, &start_hash
, &end_hash
);
679 if (start_hash
.GetHashStringList() != StartHashes
)
680 _error
->Error("The input file hadn't the expected hash!");
683 patch
.apply_against_file(out
, inp
, nullptr, &end_hash
);
688 if (_error
->PendingError() == true) {
689 std::cerr
<< "FAILED to read or write files" << std::endl
;
694 std::clog
<< "rred: finished file patching of " << Path
<< "." << std::endl
;
697 struct stat bufbase
, bufpatch
;
698 if (stat(Path
.c_str(), &bufbase
) != 0 ||
699 stat(patch_name
.c_str(), &bufpatch
) != 0)
700 return _error
->Errno("stat", _("Failed to stat %s"), Path
.c_str());
702 struct timeval times
[2];
703 times
[0].tv_sec
= bufbase
.st_atime
;
704 times
[1].tv_sec
= bufpatch
.st_mtime
;
705 times
[0].tv_usec
= times
[1].tv_usec
= 0;
706 if (utimes(Itm
->DestFile
.c_str(), times
) != 0)
707 return _error
->Errno("utimes",_("Failed to set modification time"));
709 if (stat(Itm
->DestFile
.c_str(), &bufbase
) != 0)
710 return _error
->Errno("stat", _("Failed to stat %s"), Itm
->DestFile
.c_str());
712 Res
.LastModified
= bufbase
.st_mtime
;
713 Res
.Size
= bufbase
.st_size
;
714 Res
.TakeHashes(end_hash
);
721 RredMethod() : aptMethod("rred", "2.0", SendConfig
), Debug(false) {}
724 int main(int argc
, char **argv
)
727 bool just_diff
= true;
732 return RredMethod().Run();
735 // Usage: rred -t input output diff ...
736 if (argc
> 1 && strcmp(argv
[1], "-t") == 0) {
737 // Read config files so we see compressors.
738 pkgInitConfig(*_config
);
742 } else if (argc
> 1 && strcmp(argv
[1], "-f") == 0) {
749 for (; i
< argc
; i
++) {
751 if (p
.Open(argv
[i
], FileFd::ReadOnly
) == false) {
752 _error
->DumpErrors(std::cerr
);
755 if (patch
.read_diff(p
, NULL
) == false)
757 _error
->DumpErrors(std::cerr
);
764 std::cerr
<< "Patching " << argv
[2] << " into " << argv
[3] << "\n";
765 inp
.Open(argv
[2], FileFd::ReadOnly
,FileFd::Extension
);
766 out
.Open(argv
[3], FileFd::WriteOnly
| FileFd::Create
| FileFd::Empty
| FileFd::BufferedWrite
, FileFd::Extension
);
767 patch
.apply_against_file(out
, inp
);
769 } else if (just_diff
) {
771 out
.OpenDescriptor(STDOUT_FILENO
, FileFd::WriteOnly
| FileFd::Create
);
772 patch
.write_diff(out
);
776 out
.OpenDescriptor(STDOUT_FILENO
, FileFd::WriteOnly
| FileFd::Create
| FileFd::BufferedWrite
);
777 inp
.OpenDescriptor(STDIN_FILENO
, FileFd::ReadOnly
);
778 patch
.apply_against_file(out
, inp
);