1 // Copyright (c) 2014 Anthony Towns
3 // This program is free software; you can redistribute it and/or modify
4 // it under the terms of the GNU General Public License as published by
5 // the Free Software Foundation; either version 2 of the License, or
6 // (at your option) any later version.
10 #include <apt-pkg/fileutl.h>
11 #include <apt-pkg/error.h>
12 #include <apt-pkg/acquire-method.h>
13 #include <apt-pkg/strutl.h>
14 #include <apt-pkg/hashes.h>
15 #include <apt-pkg/configuration.h>
33 #define BLOCK_SIZE (512*1024)
41 MemBlock(size_t size
) : size(size
), next(NULL
)
43 free
= start
= new char[size
];
46 size_t avail(void) { return size
- (free
- start
); }
51 free
= start
= new char[BLOCK_SIZE
];
67 char *add_easy(char *src
, size_t len
, char *last
)
70 for (MemBlock
*k
= this; k
; k
= k
->next
) {
71 if (k
->free
== last
) {
72 if (len
<= k
->avail()) {
73 char *n
= k
->add(src
, len
);
81 } else if (last
>= start
&& last
< free
) {
89 char *add(char *src
, size_t len
) {
92 if (len
> BLOCK_SIZE
) {
93 next
= new MemBlock(len
);
98 return next
->add(src
, len
);
102 memcpy(dst
, src
, len
);
110 * 1. write out <offset> lines unchanged
111 * 2. skip <del_cnt> lines from source
112 * 3. write out <add_cnt> lines (<add>/<add_len>)
116 size_t add_cnt
; /* lines */
117 size_t add_len
; /* bytes */
123 del_cnt
= add_cnt
= add_len
= 0;
127 /* actually, don't write <lines> lines from <add> */
128 void skip_lines(size_t lines
)
131 char *s
= (char*) memchr(add
, '\n', add_len
);
134 add_len
-= (s
- add
);
139 assert(add_cnt
== 0);
150 std::list
<struct Change
> changes
;
151 std::list
<struct Change
>::iterator where
;
152 size_t pos
; // line number is as far left of iterator as possible
154 bool pos_is_okay(void) const
158 std::list
<struct Change
>::const_iterator x
;
159 for (x
= changes
.begin(); x
!= where
; ++x
) {
160 assert(x
!= changes
.end());
161 cpos
+= x
->offset
+ x
->add_cnt
;
171 where
= changes
.end();
175 std::list
<struct Change
>::iterator
begin(void) { return changes
.begin(); }
176 std::list
<struct Change
>::iterator
end(void) { return changes
.end(); }
178 std::list
<struct Change
>::reverse_iterator
rbegin(void) { return changes
.rbegin(); }
179 std::list
<struct Change
>::reverse_iterator
rend(void) { return changes
.rend(); }
181 void add_change(Change c
) {
182 assert(pos_is_okay());
183 go_to_change_for(c
.offset
);
184 assert(pos
+ where
->offset
== c
.offset
);
186 delete_lines(c
.del_cnt
);
187 assert(pos
+ where
->offset
== c
.offset
);
189 assert(pos_is_okay());
190 if (where
->add_len
> 0)
192 assert(where
->add_len
== 0 && where
->add_cnt
== 0);
194 where
->add_len
= c
.add_len
;
195 where
->add_cnt
= c
.add_cnt
;
198 assert(pos_is_okay());
200 assert(pos_is_okay());
206 while (where
->offset
== 0 && where
!= changes
.begin()) {
209 std::list
<struct Change
>::iterator next
= where
;
212 while (next
!= changes
.end() && next
->offset
== 0) {
213 where
->del_cnt
+= next
->del_cnt
;
215 if (next
->add
== NULL
) {
216 next
= changes
.erase(next
);
217 } else if (where
->add
== NULL
) {
218 where
->add
= next
->add
;
219 where
->add_len
= next
->add_len
;
220 where
->add_cnt
= next
->add_cnt
;
221 next
= changes
.erase(next
);
228 void go_to_change_for(size_t line
)
230 while(where
!= changes
.end()) {
235 if (pos
+ where
->offset
+ where
->add_cnt
<= line
) {
239 // line is somewhere in this slot
240 if (line
< pos
+ where
->offset
) {
242 } else if (line
== pos
+ where
->offset
) {
250 /* it goes before this patch */
254 void new_change(void) { insert(where
->offset
); }
256 void insert(size_t offset
)
258 assert(pos_is_okay());
259 assert(where
== changes
.end() || offset
<= where
->offset
);
260 if (where
!= changes
.end())
261 where
->offset
-= offset
;
262 changes
.insert(where
, Change(offset
));
264 assert(pos_is_okay());
267 void split(size_t offset
)
269 assert(pos_is_okay());
271 assert(where
->offset
< offset
);
272 assert(offset
< where
->offset
+ where
->add_cnt
);
274 size_t keep_lines
= offset
- where
->offset
;
276 Change
before(*where
);
280 where
->skip_lines(keep_lines
);
282 before
.add_cnt
= keep_lines
;
283 before
.add_len
-= where
->add_len
;
285 changes
.insert(where
, before
);
287 assert(pos_is_okay());
290 void delete_lines(size_t cnt
)
292 std::list
<struct Change
>::iterator x
= where
;
293 assert(pos_is_okay());
304 if (x
== changes
.end()) {
312 where
->del_cnt
+= del
;
315 assert(pos_is_okay());
319 assert(pos_is_okay());
321 pos
-= where
->offset
+ where
->add_cnt
;
322 assert(pos_is_okay());
326 assert(pos_is_okay());
327 pos
+= where
->offset
+ where
->add_cnt
;
329 assert(pos_is_okay());
334 FileChanges filechanges
;
337 static bool retry_fwrite(char *b
, size_t l
, FileFd
&f
, Hashes
*hash
)
339 if (f
.Write(b
, l
) == false)
342 hash
->Add((unsigned char*)b
, l
);
346 static void dump_rest(FileFd
&o
, FileFd
&i
, Hashes
*hash
)
348 char buffer
[BLOCK_SIZE
];
349 unsigned long long l
= 0;
350 while (i
.Read(buffer
, sizeof(buffer
), &l
)) {
351 if (l
==0 || !retry_fwrite(buffer
, l
, o
, hash
))
356 static void dump_lines(FileFd
&o
, FileFd
&i
, size_t n
, Hashes
*hash
)
358 char buffer
[BLOCK_SIZE
];
360 if (i
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
362 size_t const l
= strlen(buffer
);
363 if (l
== 0 || buffer
[l
-1] == '\n')
365 retry_fwrite(buffer
, l
, o
, hash
);
369 static void skip_lines(FileFd
&i
, int n
)
371 char buffer
[BLOCK_SIZE
];
373 if (i
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
375 size_t const l
= strlen(buffer
);
376 if (l
== 0 || buffer
[l
-1] == '\n')
381 static void dump_mem(FileFd
&o
, char *p
, size_t s
, Hashes
*hash
) {
382 retry_fwrite(p
, s
, o
, hash
);
387 bool read_diff(FileFd
&f
, Hashes
* const h
)
389 char buffer
[BLOCK_SIZE
];
390 bool cmdwanted
= true;
392 Change
ch(std::numeric_limits
<size_t>::max());
393 if (f
.ReadLine(buffer
, sizeof(buffer
)) == NULL
)
394 return _error
->Error("Reading first line of patchfile %s failed", f
.Name().c_str());
402 s
= strtoul(buffer
, &m
, 10);
403 if (unlikely(m
== buffer
|| s
== std::numeric_limits
<unsigned long>::max() || errno
!= 0))
404 return _error
->Error("Parsing patchfile %s failed: Expected an effected line start", f
.Name().c_str());
405 else if (*m
== ',') {
407 e
= strtol(m
, &c
, 10);
408 if (unlikely(m
== c
|| e
== std::numeric_limits
<unsigned long>::max() || errno
!= 0))
409 return _error
->Error("Parsing patchfile %s failed: Expected an effected line end", f
.Name().c_str());
411 return _error
->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f
.Name().c_str(), e
, s
);
417 return _error
->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f
.Name().c_str());
428 if (unlikely(s
== 0))
429 return _error
->Error("Parsing patchfile %s failed: Change command can't effect line zero", f
.Name().c_str());
435 ch
.del_cnt
= e
- s
+ 1;
438 if (unlikely(s
== 0))
439 return _error
->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f
.Name().c_str());
441 ch
.del_cnt
= e
- s
+ 1;
445 filechanges
.add_change(ch
);
448 return _error
->Error("Parsing patchfile %s failed: Unknown command", f
.Name().c_str());
450 } else { /* !cmdwanted */
451 if (strcmp(buffer
, ".\n") == 0) {
453 filechanges
.add_change(ch
);
459 last
= ch
.add
+ ch
.add_len
;
461 add
= add_text
.add_easy(buffer
, l
, last
);
467 filechanges
.add_change(ch
);
470 ch
.offset
+= ch
.add_cnt
;
477 } while(f
.ReadLine(buffer
, sizeof(buffer
)));
481 void write_diff(FileFd
&f
)
483 unsigned long long line
= 0;
484 std::list
<struct Change
>::reverse_iterator ch
;
485 for (ch
= filechanges
.rbegin(); ch
!= filechanges
.rend(); ++ch
) {
486 line
+= ch
->offset
+ ch
->del_cnt
;
489 for (ch
= filechanges
.rbegin(); ch
!= filechanges
.rend(); ++ch
) {
490 std::list
<struct Change
>::reverse_iterator mg_i
, mg_e
= ch
;
491 while (ch
->del_cnt
== 0 && ch
->offset
== 0)
495 if (ch
->add_cnt
> 0) {
496 if (ch
->del_cnt
== 0) {
497 strprintf(buf
, "%llua\n", line
);
498 } else if (ch
->del_cnt
== 1) {
499 strprintf(buf
, "%lluc\n", line
+1);
501 strprintf(buf
, "%llu,%lluc\n", line
+1, line
+ch
->del_cnt
);
503 f
.Write(buf
.c_str(), buf
.length());
507 dump_mem(f
, mg_i
->add
, mg_i
->add_len
, NULL
);
508 } while (mg_i
-- != mg_e
);
511 f
.Write(buf
.c_str(), buf
.length());
512 } else if (ch
->del_cnt
== 1) {
513 strprintf(buf
, "%llud\n", line
+1);
514 f
.Write(buf
.c_str(), buf
.length());
515 } else if (ch
->del_cnt
> 1) {
516 strprintf(buf
, "%llu,%llud\n", line
+1, line
+ch
->del_cnt
);
517 f
.Write(buf
.c_str(), buf
.length());
523 void apply_against_file(FileFd
&out
, FileFd
&in
, Hashes
*hash
= NULL
)
525 std::list
<struct Change
>::iterator ch
;
526 for (ch
= filechanges
.begin(); ch
!= filechanges
.end(); ++ch
) {
527 dump_lines(out
, in
, ch
->offset
, hash
);
528 skip_lines(in
, ch
->del_cnt
);
529 dump_mem(out
, ch
->add
, ch
->add_len
, hash
);
531 dump_rest(out
, in
, hash
);
535 class RredMethod
: public pkgAcqMethod
{
540 std::string FileName
;
541 HashStringList ExpectedHashes
;
542 PDiffFile(std::string
const &FileName
, HashStringList
const &ExpectedHashes
) :
543 FileName(FileName
), ExpectedHashes(ExpectedHashes
) {}
546 HashStringList
ReadExpectedHashesForPatch(unsigned int const patch
, std::string
const &Message
)
548 HashStringList ExpectedHashes
;
549 for (char const * const * type
= HashString::SupportedHashes(); *type
!= NULL
; ++type
)
552 strprintf(tagname
, "Patch-%d-%s-Hash", patch
, *type
);
553 std::string
const hashsum
= LookupTag(Message
, tagname
.c_str());
554 if (hashsum
.empty() == false)
555 ExpectedHashes
.push_back(HashString(*type
, hashsum
));
557 return ExpectedHashes
;
561 virtual bool URIAcquire(std::string
const &Message
, FetchItem
*Itm
) APT_OVERRIDE
{
562 Debug
= _config
->FindB("Debug::pkgAcquire::RRed", false);
564 std::string Path
= Get
.Host
+ Get
.Path
; // rred:/path - no host
567 Res
.Filename
= Itm
->DestFile
;
568 if (Itm
->Uri
.empty())
570 Path
= Itm
->DestFile
;
571 Itm
->DestFile
.append(".result");
575 std::vector
<PDiffFile
> patchfiles
;
578 if (FileExists(Path
+ ".ed") == true)
580 HashStringList
const ExpectedHashes
= ReadExpectedHashesForPatch(0, Message
);
581 std::string
const FileName
= Path
+ ".ed";
582 if (ExpectedHashes
.usable() == false)
583 return _error
->Error("No hashes found for uncompressed patch: %s", FileName
.c_str());
584 patchfiles
.push_back(PDiffFile(FileName
, ExpectedHashes
));
588 _error
->PushToStack();
589 std::vector
<std::string
> patches
= GetListOfFilesInDir(flNotFile(Path
), "gz", true, false);
590 _error
->RevertToStack();
592 std::string
const baseName
= Path
+ ".ed.";
593 unsigned int seen_patches
= 0;
594 for (std::vector
<std::string
>::const_iterator p
= patches
.begin();
595 p
!= patches
.end(); ++p
)
597 if (p
->compare(0, baseName
.length(), baseName
) == 0)
599 HashStringList
const ExpectedHashes
= ReadExpectedHashesForPatch(seen_patches
, Message
);
600 if (ExpectedHashes
.usable() == false)
601 return _error
->Error("No hashes found for uncompressed patch %d: %s", seen_patches
, p
->c_str());
602 patchfiles
.push_back(PDiffFile(*p
, ExpectedHashes
));
608 std::string patch_name
;
609 for (std::vector
<PDiffFile
>::iterator I
= patchfiles
.begin();
610 I
!= patchfiles
.end();
613 patch_name
= I
->FileName
;
615 std::clog
<< "Patching " << Path
<< " with " << patch_name
619 Hashes
patch_hash(I
->ExpectedHashes
);
620 // all patches are compressed, even if the name doesn't reflect it
621 if (p
.Open(patch_name
, FileFd::ReadOnly
, FileFd::Gzip
) == false ||
622 patch
.read_diff(p
, &patch_hash
) == false)
624 _error
->DumpErrors(std::cerr
, GlobalError::DEBUG
, false);
628 HashStringList
const hsl
= patch_hash
.GetHashStringList();
629 if (hsl
!= I
->ExpectedHashes
)
630 return _error
->Error("Hash Sum mismatch for uncompressed patch %s", patch_name
.c_str());
634 std::clog
<< "Applying patches against " << Path
635 << " and writing results to " << Itm
->DestFile
639 if (inp
.Open(Path
, FileFd::ReadOnly
, FileFd::Extension
) == false)
641 std::cerr
<< "FAILED to open inp " << Path
<< std::endl
;
642 return _error
->Error("Failed to open inp %s", Path
.c_str());
644 if (out
.Open(Itm
->DestFile
, FileFd::WriteOnly
| FileFd::Create
, FileFd::Extension
) == false)
646 std::cerr
<< "FAILED to open out " << Itm
->DestFile
<< std::endl
;
647 return _error
->Error("Failed to open out %s", Itm
->DestFile
.c_str());
650 Hashes
hash(Itm
->ExpectedHashes
);
651 patch
.apply_against_file(out
, inp
, &hash
);
657 std::clog
<< "rred: finished file patching of " << Path
<< "." << std::endl
;
660 struct stat bufbase
, bufpatch
;
661 if (stat(Path
.c_str(), &bufbase
) != 0 ||
662 stat(patch_name
.c_str(), &bufpatch
) != 0)
663 return _error
->Errno("stat", _("Failed to stat"));
665 struct timeval times
[2];
666 times
[0].tv_sec
= bufbase
.st_atime
;
667 times
[1].tv_sec
= bufpatch
.st_mtime
;
668 times
[0].tv_usec
= times
[1].tv_usec
= 0;
669 if (utimes(Itm
->DestFile
.c_str(), times
) != 0)
670 return _error
->Errno("utimes",_("Failed to set modification time"));
672 if (stat(Itm
->DestFile
.c_str(), &bufbase
) != 0)
673 return _error
->Errno("stat", _("Failed to stat"));
675 Res
.LastModified
= bufbase
.st_mtime
;
676 Res
.Size
= bufbase
.st_size
;
677 Res
.TakeHashes(hash
);
683 bool Configuration(std::string Message
) APT_OVERRIDE
685 if (pkgAcqMethod::Configuration(Message
) == false)
694 RredMethod() : pkgAcqMethod("2.0",SingleInstance
| SendConfig
), Debug(false) {}
697 int main(int argc
, char **argv
)
700 bool just_diff
= true;
708 if (argc
> 1 && strcmp(argv
[1], "-f") == 0) {
715 for (; i
< argc
; i
++) {
717 if (p
.Open(argv
[i
], FileFd::ReadOnly
) == false) {
718 _error
->DumpErrors(std::cerr
);
721 if (patch
.read_diff(p
, NULL
) == false)
723 _error
->DumpErrors(std::cerr
);
730 out
.OpenDescriptor(STDOUT_FILENO
, FileFd::WriteOnly
| FileFd::Create
);
731 patch
.write_diff(out
);
734 out
.OpenDescriptor(STDOUT_FILENO
, FileFd::WriteOnly
| FileFd::Create
);
735 inp
.OpenDescriptor(STDIN_FILENO
, FileFd::ReadOnly
);
736 patch
.apply_against_file(out
, inp
);