]> git.saurik.com Git - apt.git/blob - methods/rred.cc
dpkgpm: Use ptsname_r() instead of ptsname() to be thread-safe
[apt.git] / methods / rred.cc
1 // Copyright (c) 2014 Anthony Towns
2 //
3 // This program is free software; you can redistribute it and/or modify
4 // it under the terms of the GNU General Public License as published by
5 // the Free Software Foundation; either version 2 of the License, or
6 // (at your option) any later version.
7
8 #include <config.h>
9
10 #include <apt-pkg/fileutl.h>
11 #include <apt-pkg/error.h>
12 #include <apt-pkg/acquire-method.h>
13 #include <apt-pkg/strutl.h>
14 #include <apt-pkg/hashes.h>
15 #include <apt-pkg/configuration.h>
16
17 #include <stddef.h>
18 #include <iostream>
19 #include <string>
20 #include <list>
21 #include <vector>
22
23 #include <assert.h>
24 #include <errno.h>
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <sys/stat.h>
29 #include <sys/time.h>
30
31 #include <apti18n.h>
32
33 #define BLOCK_SIZE (512*1024)
34
35 class MemBlock {
36 char *start;
37 size_t size;
38 char *free;
39 MemBlock *next;
40
41 MemBlock(size_t size) : size(size), next(NULL)
42 {
43 free = start = new char[size];
44 }
45
46 size_t avail(void) { return size - (free - start); }
47
48 public:
49
50 MemBlock(void) {
51 free = start = new char[BLOCK_SIZE];
52 size = BLOCK_SIZE;
53 next = NULL;
54 }
55
56 ~MemBlock() {
57 delete [] start;
58 delete next;
59 }
60
61 void clear(void) {
62 free = start;
63 if (next)
64 next->clear();
65 }
66
67 char *add_easy(char *src, size_t len, char *last)
68 {
69 if (last) {
70 for (MemBlock *k = this; k; k = k->next) {
71 if (k->free == last) {
72 if (len <= k->avail()) {
73 char *n = k->add(src, len);
74 assert(last == n);
75 if (last == n)
76 return NULL;
77 return n;
78 } else {
79 break;
80 }
81 } else if (last >= start && last < free) {
82 break;
83 }
84 }
85 }
86 return add(src, len);
87 }
88
89 char *add(char *src, size_t len) {
90 if (len > avail()) {
91 if (!next) {
92 if (len > BLOCK_SIZE) {
93 next = new MemBlock(len);
94 } else {
95 next = new MemBlock;
96 }
97 }
98 return next->add(src, len);
99 }
100 char *dst = free;
101 free += len;
102 memcpy(dst, src, len);
103 return dst;
104 }
105 };
106
107 struct Change {
108 /* Ordering:
109 *
110 * 1. write out <offset> lines unchanged
111 * 2. skip <del_cnt> lines from source
112 * 3. write out <add_cnt> lines (<add>/<add_len>)
113 */
114 size_t offset;
115 size_t del_cnt;
116 size_t add_cnt; /* lines */
117 size_t add_len; /* bytes */
118 char *add;
119
120 Change(size_t off)
121 {
122 offset = off;
123 del_cnt = add_cnt = add_len = 0;
124 add = NULL;
125 }
126
127 /* actually, don't write <lines> lines from <add> */
128 void skip_lines(size_t lines)
129 {
130 while (lines > 0) {
131 char *s = (char*) memchr(add, '\n', add_len);
132 assert(s != NULL);
133 s++;
134 add_len -= (s - add);
135 add_cnt--;
136 lines--;
137 if (add_len == 0) {
138 add = NULL;
139 assert(add_cnt == 0);
140 assert(lines == 0);
141 } else {
142 add = s;
143 assert(add_cnt > 0);
144 }
145 }
146 }
147 };
148
149 class FileChanges {
150 std::list<struct Change> changes;
151 std::list<struct Change>::iterator where;
152 size_t pos; // line number is as far left of iterator as possible
153
154 bool pos_is_okay(void) const
155 {
156 #ifdef POSDEBUG
157 size_t cpos = 0;
158 std::list<struct Change>::const_iterator x;
159 for (x = changes.begin(); x != where; ++x) {
160 assert(x != changes.end());
161 cpos += x->offset + x->add_cnt;
162 }
163 return cpos == pos;
164 #else
165 return true;
166 #endif
167 }
168
169 public:
170 FileChanges() {
171 where = changes.end();
172 pos = 0;
173 }
174
175 std::list<struct Change>::iterator begin(void) { return changes.begin(); }
176 std::list<struct Change>::iterator end(void) { return changes.end(); }
177
178 std::list<struct Change>::reverse_iterator rbegin(void) { return changes.rbegin(); }
179 std::list<struct Change>::reverse_iterator rend(void) { return changes.rend(); }
180
181 void add_change(Change c) {
182 assert(pos_is_okay());
183 go_to_change_for(c.offset);
184 assert(pos + where->offset == c.offset);
185 if (c.del_cnt > 0)
186 delete_lines(c.del_cnt);
187 assert(pos + where->offset == c.offset);
188 if (c.add_len > 0) {
189 assert(pos_is_okay());
190 if (where->add_len > 0)
191 new_change();
192 assert(where->add_len == 0 && where->add_cnt == 0);
193
194 where->add_len = c.add_len;
195 where->add_cnt = c.add_cnt;
196 where->add = c.add;
197 }
198 assert(pos_is_okay());
199 merge();
200 assert(pos_is_okay());
201 }
202
203 private:
204 void merge(void)
205 {
206 while (where->offset == 0 && where != changes.begin()) {
207 left();
208 }
209 std::list<struct Change>::iterator next = where;
210 ++next;
211
212 while (next != changes.end() && next->offset == 0) {
213 where->del_cnt += next->del_cnt;
214 next->del_cnt = 0;
215 if (next->add == NULL) {
216 next = changes.erase(next);
217 } else if (where->add == NULL) {
218 where->add = next->add;
219 where->add_len = next->add_len;
220 where->add_cnt = next->add_cnt;
221 next = changes.erase(next);
222 } else {
223 ++next;
224 }
225 }
226 }
227
228 void go_to_change_for(size_t line)
229 {
230 while(where != changes.end()) {
231 if (line < pos) {
232 left();
233 continue;
234 }
235 if (pos + where->offset + where->add_cnt <= line) {
236 right();
237 continue;
238 }
239 // line is somewhere in this slot
240 if (line < pos + where->offset) {
241 break;
242 } else if (line == pos + where->offset) {
243 return;
244 } else {
245 split(line - pos);
246 right();
247 return;
248 }
249 }
250 /* it goes before this patch */
251 insert(line-pos);
252 }
253
254 void new_change(void) { insert(where->offset); }
255
256 void insert(size_t offset)
257 {
258 assert(pos_is_okay());
259 assert(where == changes.end() || offset <= where->offset);
260 if (where != changes.end())
261 where->offset -= offset;
262 changes.insert(where, Change(offset));
263 --where;
264 assert(pos_is_okay());
265 }
266
267 void split(size_t offset)
268 {
269 assert(pos_is_okay());
270
271 assert(where->offset < offset);
272 assert(offset < where->offset + where->add_cnt);
273
274 size_t keep_lines = offset - where->offset;
275
276 Change before(*where);
277
278 where->del_cnt = 0;
279 where->offset = 0;
280 where->skip_lines(keep_lines);
281
282 before.add_cnt = keep_lines;
283 before.add_len -= where->add_len;
284
285 changes.insert(where, before);
286 --where;
287 assert(pos_is_okay());
288 }
289
290 void delete_lines(size_t cnt)
291 {
292 std::list<struct Change>::iterator x = where;
293 assert(pos_is_okay());
294 while (cnt > 0)
295 {
296 size_t del;
297 del = x->add_cnt;
298 if (del > cnt)
299 del = cnt;
300 x->skip_lines(del);
301 cnt -= del;
302
303 ++x;
304 if (x == changes.end()) {
305 del = cnt;
306 } else {
307 del = x->offset;
308 if (del > cnt)
309 del = cnt;
310 x->offset -= del;
311 }
312 where->del_cnt += del;
313 cnt -= del;
314 }
315 assert(pos_is_okay());
316 }
317
318 void left(void) {
319 assert(pos_is_okay());
320 --where;
321 pos -= where->offset + where->add_cnt;
322 assert(pos_is_okay());
323 }
324
325 void right(void) {
326 assert(pos_is_okay());
327 pos += where->offset + where->add_cnt;
328 ++where;
329 assert(pos_is_okay());
330 }
331 };
332
333 class Patch {
334 FileChanges filechanges;
335 MemBlock add_text;
336
337 static bool retry_fwrite(char *b, size_t l, FileFd &f, Hashes *hash)
338 {
339 if (f.Write(b, l) == false)
340 return false;
341 if (hash)
342 hash->Add((unsigned char*)b, l);
343 return true;
344 }
345
346 static void dump_rest(FileFd &o, FileFd &i, Hashes *hash)
347 {
348 char buffer[BLOCK_SIZE];
349 unsigned long long l = 0;
350 while (i.Read(buffer, sizeof(buffer), &l)) {
351 if (l ==0 || !retry_fwrite(buffer, l, o, hash))
352 break;
353 }
354 }
355
356 static void dump_lines(FileFd &o, FileFd &i, size_t n, Hashes *hash)
357 {
358 char buffer[BLOCK_SIZE];
359 while (n > 0) {
360 if (i.ReadLine(buffer, sizeof(buffer)) == NULL)
361 buffer[0] = '\0';
362 size_t const l = strlen(buffer);
363 if (l == 0 || buffer[l-1] == '\n')
364 n--;
365 retry_fwrite(buffer, l, o, hash);
366 }
367 }
368
369 static void skip_lines(FileFd &i, int n)
370 {
371 char buffer[BLOCK_SIZE];
372 while (n > 0) {
373 if (i.ReadLine(buffer, sizeof(buffer)) == NULL)
374 buffer[0] = '\0';
375 size_t const l = strlen(buffer);
376 if (l == 0 || buffer[l-1] == '\n')
377 n--;
378 }
379 }
380
381 static void dump_mem(FileFd &o, char *p, size_t s, Hashes *hash) {
382 retry_fwrite(p, s, o, hash);
383 }
384
385 public:
386
387 bool read_diff(FileFd &f, Hashes * const h)
388 {
389 char buffer[BLOCK_SIZE];
390 bool cmdwanted = true;
391
392 Change ch(std::numeric_limits<size_t>::max());
393 if (f.ReadLine(buffer, sizeof(buffer)) == NULL)
394 return _error->Error("Reading first line of patchfile %s failed", f.Name().c_str());
395 do {
396 if (h != NULL)
397 h->Add(buffer);
398 if (cmdwanted) {
399 char *m, *c;
400 size_t s, e;
401 errno = 0;
402 s = strtoul(buffer, &m, 10);
403 if (unlikely(m == buffer || s == std::numeric_limits<unsigned long>::max() || errno != 0))
404 return _error->Error("Parsing patchfile %s failed: Expected an effected line start", f.Name().c_str());
405 else if (*m == ',') {
406 ++m;
407 e = strtol(m, &c, 10);
408 if (unlikely(m == c || e == std::numeric_limits<unsigned long>::max() || errno != 0))
409 return _error->Error("Parsing patchfile %s failed: Expected an effected line end", f.Name().c_str());
410 if (unlikely(e < s))
411 return _error->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f.Name().c_str(), e, s);
412 } else {
413 e = s;
414 c = m;
415 }
416 if (s > ch.offset)
417 return _error->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f.Name().c_str());
418 switch(*c) {
419 case 'a':
420 cmdwanted = false;
421 ch.add = NULL;
422 ch.add_cnt = 0;
423 ch.add_len = 0;
424 ch.offset = s;
425 ch.del_cnt = 0;
426 break;
427 case 'c':
428 if (unlikely(s == 0))
429 return _error->Error("Parsing patchfile %s failed: Change command can't effect line zero", f.Name().c_str());
430 cmdwanted = false;
431 ch.add = NULL;
432 ch.add_cnt = 0;
433 ch.add_len = 0;
434 ch.offset = s - 1;
435 ch.del_cnt = e - s + 1;
436 break;
437 case 'd':
438 if (unlikely(s == 0))
439 return _error->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f.Name().c_str());
440 ch.offset = s - 1;
441 ch.del_cnt = e - s + 1;
442 ch.add = NULL;
443 ch.add_cnt = 0;
444 ch.add_len = 0;
445 filechanges.add_change(ch);
446 break;
447 default:
448 return _error->Error("Parsing patchfile %s failed: Unknown command", f.Name().c_str());
449 }
450 } else { /* !cmdwanted */
451 if (strcmp(buffer, ".\n") == 0) {
452 cmdwanted = true;
453 filechanges.add_change(ch);
454 } else {
455 char *last = NULL;
456 char *add;
457 size_t l;
458 if (ch.add)
459 last = ch.add + ch.add_len;
460 l = strlen(buffer);
461 add = add_text.add_easy(buffer, l, last);
462 if (!add) {
463 ch.add_len += l;
464 ch.add_cnt++;
465 } else {
466 if (ch.add) {
467 filechanges.add_change(ch);
468 ch.del_cnt = 0;
469 }
470 ch.offset += ch.add_cnt;
471 ch.add = add;
472 ch.add_len = l;
473 ch.add_cnt = 1;
474 }
475 }
476 }
477 } while(f.ReadLine(buffer, sizeof(buffer)));
478 return true;
479 }
480
481 void write_diff(FileFd &f)
482 {
483 unsigned long long line = 0;
484 std::list<struct Change>::reverse_iterator ch;
485 for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) {
486 line += ch->offset + ch->del_cnt;
487 }
488
489 for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) {
490 std::list<struct Change>::reverse_iterator mg_i, mg_e = ch;
491 while (ch->del_cnt == 0 && ch->offset == 0)
492 ++ch;
493 line -= ch->del_cnt;
494 std::string buf;
495 if (ch->add_cnt > 0) {
496 if (ch->del_cnt == 0) {
497 strprintf(buf, "%llua\n", line);
498 } else if (ch->del_cnt == 1) {
499 strprintf(buf, "%lluc\n", line+1);
500 } else {
501 strprintf(buf, "%llu,%lluc\n", line+1, line+ch->del_cnt);
502 }
503 f.Write(buf.c_str(), buf.length());
504
505 mg_i = ch;
506 do {
507 dump_mem(f, mg_i->add, mg_i->add_len, NULL);
508 } while (mg_i-- != mg_e);
509
510 buf = ".\n";
511 f.Write(buf.c_str(), buf.length());
512 } else if (ch->del_cnt == 1) {
513 strprintf(buf, "%llud\n", line+1);
514 f.Write(buf.c_str(), buf.length());
515 } else if (ch->del_cnt > 1) {
516 strprintf(buf, "%llu,%llud\n", line+1, line+ch->del_cnt);
517 f.Write(buf.c_str(), buf.length());
518 }
519 line -= ch->offset;
520 }
521 }
522
523 void apply_against_file(FileFd &out, FileFd &in, Hashes *hash = NULL)
524 {
525 std::list<struct Change>::iterator ch;
526 for (ch = filechanges.begin(); ch != filechanges.end(); ++ch) {
527 dump_lines(out, in, ch->offset, hash);
528 skip_lines(in, ch->del_cnt);
529 dump_mem(out, ch->add, ch->add_len, hash);
530 }
531 dump_rest(out, in, hash);
532 }
533 };
534
535 class RredMethod : public pkgAcqMethod {
536 private:
537 bool Debug;
538
539 struct PDiffFile {
540 std::string FileName;
541 HashStringList ExpectedHashes;
542 PDiffFile(std::string const &FileName, HashStringList const &ExpectedHashes) :
543 FileName(FileName), ExpectedHashes(ExpectedHashes) {}
544 };
545
546 HashStringList ReadExpectedHashesForPatch(unsigned int const patch, std::string const &Message)
547 {
548 HashStringList ExpectedHashes;
549 for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
550 {
551 std::string tagname;
552 strprintf(tagname, "Patch-%d-%s-Hash", patch, *type);
553 std::string const hashsum = LookupTag(Message, tagname.c_str());
554 if (hashsum.empty() == false)
555 ExpectedHashes.push_back(HashString(*type, hashsum));
556 }
557 return ExpectedHashes;
558 }
559
560 protected:
561 virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE {
562 Debug = _config->FindB("Debug::pkgAcquire::RRed", false);
563 URI Get = Itm->Uri;
564 std::string Path = Get.Host + Get.Path; // rred:/path - no host
565
566 FetchResult Res;
567 Res.Filename = Itm->DestFile;
568 if (Itm->Uri.empty())
569 {
570 Path = Itm->DestFile;
571 Itm->DestFile.append(".result");
572 } else
573 URIStart(Res);
574
575 std::vector<PDiffFile> patchfiles;
576 Patch patch;
577
578 if (FileExists(Path + ".ed") == true)
579 {
580 HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(0, Message);
581 std::string const FileName = Path + ".ed";
582 if (ExpectedHashes.usable() == false)
583 return _error->Error("No hashes found for uncompressed patch: %s", FileName.c_str());
584 patchfiles.push_back(PDiffFile(FileName, ExpectedHashes));
585 }
586 else
587 {
588 _error->PushToStack();
589 std::vector<std::string> patches = GetListOfFilesInDir(flNotFile(Path), "gz", true, false);
590 _error->RevertToStack();
591
592 std::string const baseName = Path + ".ed.";
593 unsigned int seen_patches = 0;
594 for (std::vector<std::string>::const_iterator p = patches.begin();
595 p != patches.end(); ++p)
596 {
597 if (p->compare(0, baseName.length(), baseName) == 0)
598 {
599 HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(seen_patches, Message);
600 if (ExpectedHashes.usable() == false)
601 return _error->Error("No hashes found for uncompressed patch %d: %s", seen_patches, p->c_str());
602 patchfiles.push_back(PDiffFile(*p, ExpectedHashes));
603 ++seen_patches;
604 }
605 }
606 }
607
608 std::string patch_name;
609 for (std::vector<PDiffFile>::iterator I = patchfiles.begin();
610 I != patchfiles.end();
611 ++I)
612 {
613 patch_name = I->FileName;
614 if (Debug == true)
615 std::clog << "Patching " << Path << " with " << patch_name
616 << std::endl;
617
618 FileFd p;
619 Hashes patch_hash(I->ExpectedHashes);
620 // all patches are compressed, even if the name doesn't reflect it
621 if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false ||
622 patch.read_diff(p, &patch_hash) == false)
623 {
624 _error->DumpErrors(std::cerr, GlobalError::DEBUG, false);
625 return false;
626 }
627 p.Close();
628 HashStringList const hsl = patch_hash.GetHashStringList();
629 if (hsl != I->ExpectedHashes)
630 return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
631 }
632
633 if (Debug == true)
634 std::clog << "Applying patches against " << Path
635 << " and writing results to " << Itm->DestFile
636 << std::endl;
637
638 FileFd inp, out;
639 if (inp.Open(Path, FileFd::ReadOnly, FileFd::Extension) == false)
640 {
641 std::cerr << "FAILED to open inp " << Path << std::endl;
642 return _error->Error("Failed to open inp %s", Path.c_str());
643 }
644 if (out.Open(Itm->DestFile, FileFd::WriteOnly | FileFd::Create, FileFd::Extension) == false)
645 {
646 std::cerr << "FAILED to open out " << Itm->DestFile << std::endl;
647 return _error->Error("Failed to open out %s", Itm->DestFile.c_str());
648 }
649
650 Hashes hash(Itm->ExpectedHashes);
651 patch.apply_against_file(out, inp, &hash);
652
653 out.Close();
654 inp.Close();
655
656 if (Debug == true) {
657 std::clog << "rred: finished file patching of " << Path << "." << std::endl;
658 }
659
660 struct stat bufbase, bufpatch;
661 if (stat(Path.c_str(), &bufbase) != 0 ||
662 stat(patch_name.c_str(), &bufpatch) != 0)
663 return _error->Errno("stat", _("Failed to stat"));
664
665 struct timeval times[2];
666 times[0].tv_sec = bufbase.st_atime;
667 times[1].tv_sec = bufpatch.st_mtime;
668 times[0].tv_usec = times[1].tv_usec = 0;
669 if (utimes(Itm->DestFile.c_str(), times) != 0)
670 return _error->Errno("utimes",_("Failed to set modification time"));
671
672 if (stat(Itm->DestFile.c_str(), &bufbase) != 0)
673 return _error->Errno("stat", _("Failed to stat"));
674
675 Res.LastModified = bufbase.st_mtime;
676 Res.Size = bufbase.st_size;
677 Res.TakeHashes(hash);
678 URIDone(Res);
679
680 return true;
681 }
682
683 bool Configuration(std::string Message) APT_OVERRIDE
684 {
685 if (pkgAcqMethod::Configuration(Message) == false)
686 return false;
687
688 DropPrivsOrDie();
689
690 return true;
691 }
692
693 public:
694 RredMethod() : pkgAcqMethod("2.0",SingleInstance | SendConfig), Debug(false) {}
695 };
696
697 int main(int argc, char **argv)
698 {
699 int i;
700 bool just_diff = true;
701 Patch patch;
702
703 if (argc <= 1) {
704 RredMethod Mth;
705 return Mth.Run();
706 }
707
708 if (argc > 1 && strcmp(argv[1], "-f") == 0) {
709 just_diff = false;
710 i = 2;
711 } else {
712 i = 1;
713 }
714
715 for (; i < argc; i++) {
716 FileFd p;
717 if (p.Open(argv[i], FileFd::ReadOnly) == false) {
718 _error->DumpErrors(std::cerr);
719 exit(1);
720 }
721 if (patch.read_diff(p, NULL) == false)
722 {
723 _error->DumpErrors(std::cerr);
724 exit(2);
725 }
726 }
727
728 if (just_diff) {
729 FileFd out;
730 out.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly | FileFd::Create);
731 patch.write_diff(out);
732 } else {
733 FileFd out, inp;
734 out.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly | FileFd::Create);
735 inp.OpenDescriptor(STDIN_FILENO, FileFd::ReadOnly);
736 patch.apply_against_file(out, inp);
737 }
738 return 0;
739 }