methods/rred.cc

   1 // Copyright (c) 2014 Anthony Towns
   2 //
   3 // This program is free software; you can redistribute it and/or modify
   4 // it under the terms of the GNU General Public License as published by
   5 // the Free Software Foundation; either version 2 of the License, or
   6 // (at your option) any later version.
   7
   8 #include <config.h>
   9
  10 #include <apt-pkg/fileutl.h>
  11 #include <apt-pkg/error.h>
  12 #include <apt-pkg/acquire-method.h>
  13 #include <apt-pkg/strutl.h>
  14 #include <apt-pkg/hashes.h>
  15 #include <apt-pkg/configuration.h>
  16
  17 #include <stddef.h>
  18 #include <iostream>
  19 #include <string>
  20 #include <list>
  21 #include <vector>
  22
  23 #include <assert.h>
  24 #include <errno.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <string.h>
  28 #include <sys/stat.h>
  29 #include <sys/time.h>
  30
  31 #include <apti18n.h>
  32
  33 #define BLOCK_SIZE (512*1024)
  34
  35 class MemBlock {
  36    char *start;
  37    size_t size;
  38    char *free;
  39    MemBlock *next;
  40
  41    MemBlock(size_t size) : size(size), next(NULL)
  42    {
  43       free = start = new char[size];
  44    }
  45
  46    size_t avail(void) { return size - (free - start); }
  47
  48    public:
  49
  50    MemBlock(void) {
  51       free = start = new char[BLOCK_SIZE];
  52       size = BLOCK_SIZE;
  53       next = NULL;
  54    }
  55
  56    ~MemBlock() {
  57       delete [] start;
  58       delete next;
  59    }
  60
  61    void clear(void) {
  62       free = start;
  63       if (next)
  64          next->clear();
  65    }
  66
  67    char *add_easy(char *src, size_t len, char *last)
  68    {
  69       if (last) {
  70          for (MemBlock *k = this; k; k = k->next) {
  71             if (k->free == last) {
  72                if (len <= k->avail()) {
  73                   char *n = k->add(src, len);
  74                   assert(last == n);
  75                   if (last == n)
  76                      return NULL;
  77                   return n;
  78                } else {
  79                   break;
  80                }
  81             } else if (last >= start && last < free) {
  82                break;
  83             }
  84          }
  85       }
  86       return add(src, len);
  87    }
  88
  89    char *add(char *src, size_t len) {
  90       if (len > avail()) {
  91          if (!next) {
  92             if (len > BLOCK_SIZE)  {
  93                next = new MemBlock(len);
  94             } else {
  95                next = new MemBlock;
  96             }
  97          }
  98          return next->add(src, len);
  99       }
 100       char *dst = free;
 101       free += len;
 102       memcpy(dst, src, len);
 103       return dst;
 104    }
 105 };
 106
 107 struct Change {
 108    /* Ordering:
 109     *
 110     *   1. write out <offset> lines unchanged
 111     *   2. skip <del_cnt> lines from source
 112     *   3. write out <add_cnt> lines (<add>/<add_len>)
 113     */
 114    size_t offset;
 115    size_t del_cnt;
 116    size_t add_cnt; /* lines */
 117    size_t add_len; /* bytes */
 118    char *add;
 119
 120    Change(size_t off)
 121    {
 122       offset = off;
 123       del_cnt = add_cnt = add_len = 0;
 124       add = NULL;
 125    }
 126
 127    /* actually, don't write <lines> lines from <add> */
 128    void skip_lines(size_t lines)
 129    {
 130       while (lines > 0) {
 131          char *s = (char*) memchr(add, '\n', add_len);
 132          assert(s != NULL);
 133          s++;
 134          add_len -= (s - add);
 135          add_cnt--;
 136          lines--;
 137          if (add_len == 0) {
 138             add = NULL;
 139             assert(add_cnt == 0);
 140             assert(lines == 0);
 141          } else {
 142             add = s;
 143             assert(add_cnt > 0);
 144          }
 145       }
 146    }
 147 };
 148
 149 class FileChanges {
 150    std::list<struct Change> changes;
 151    std::list<struct Change>::iterator where;
 152    size_t pos; // line number is as far left of iterator as possible
 153
 154    bool pos_is_okay(void) const
 155    {
 156 #ifdef POSDEBUG
 157       size_t cpos = 0;
 158       std::list<struct Change>::const_iterator x;
 159       for (x = changes.begin(); x != where; ++x) {
 160          assert(x != changes.end());
 161          cpos += x->offset + x->add_cnt;
 162       }
 163       return cpos == pos;
 164 #else
 165       return true;
 166 #endif
 167    }
 168
 169    public:
 170    FileChanges() {
 171       where = changes.end();
 172       pos = 0;
 173    }
 174
 175    std::list<struct Change>::iterator begin(void) { return changes.begin(); }
 176    std::list<struct Change>::iterator end(void) { return changes.end(); }
 177
 178    std::list<struct Change>::reverse_iterator rbegin(void) { return changes.rbegin(); }
 179    std::list<struct Change>::reverse_iterator rend(void) { return changes.rend(); }
 180
 181    void add_change(Change c) {
 182       assert(pos_is_okay());
 183       go_to_change_for(c.offset);
 184       assert(pos + where->offset == c.offset);
 185       if (c.del_cnt > 0)
 186          delete_lines(c.del_cnt);
 187       assert(pos + where->offset == c.offset);
 188       if (c.add_len > 0) {
 189          assert(pos_is_okay());
 190          if (where->add_len > 0)
 191             new_change();
 192          assert(where->add_len == 0 && where->add_cnt == 0);
 193
 194          where->add_len = c.add_len;
 195          where->add_cnt = c.add_cnt;
 196          where->add = c.add;
 197       }
 198       assert(pos_is_okay());
 199       merge();
 200       assert(pos_is_okay());
 201    }
 202
 203    private:
 204    void merge(void)
 205    {
 206       while (where->offset == 0 && where != changes.begin()) {
 207          left();
 208       }
 209       std::list<struct Change>::iterator next = where;
 210       ++next;
 211
 212       while (next != changes.end() && next->offset == 0) {
 213          where->del_cnt += next->del_cnt;
 214          next->del_cnt = 0;
 215          if (next->add == NULL) {
 216             next = changes.erase(next);
 217          } else if (where->add == NULL) {
 218             where->add = next->add;
 219             where->add_len = next->add_len;
 220             where->add_cnt = next->add_cnt;
 221             next = changes.erase(next);
 222          } else {
 223             ++next;
 224          }
 225       }
 226    }
 227
 228    void go_to_change_for(size_t line)
 229    {
 230       while(where != changes.end()) {
 231          if (line < pos) {
 232             left();
 233             continue;
 234          }
 235          if (pos + where->offset + where->add_cnt <= line) {
 236             right();
 237             continue;
 238          }
 239          // line is somewhere in this slot
 240          if (line < pos + where->offset) {
 241             break;
 242          } else if (line == pos + where->offset) {
 243             return;
 244          } else {
 245             split(line - pos);
 246             right();
 247             return;
 248          }
 249       }
 250       /* it goes before this patch */
 251       insert(line-pos);
 252    }
 253
 254    void new_change(void) { insert(where->offset); }
 255
 256    void insert(size_t offset)
 257    {
 258       assert(pos_is_okay());
 259       assert(where == changes.end() || offset <= where->offset);
 260       if (where != changes.end())
 261          where->offset -= offset;
 262       changes.insert(where, Change(offset));
 263       --where;
 264       assert(pos_is_okay());
 265    }
 266
 267    void split(size_t offset)
 268    {
 269       assert(pos_is_okay());
 270
 271       assert(where->offset < offset);
 272       assert(offset < where->offset + where->add_cnt);
 273
 274       size_t keep_lines = offset - where->offset;
 275
 276       Change before(*where);
 277
 278       where->del_cnt = 0;
 279       where->offset = 0;
 280       where->skip_lines(keep_lines);
 281
 282       before.add_cnt = keep_lines;
 283       before.add_len -= where->add_len;
 284
 285       changes.insert(where, before);
 286       --where;
 287       assert(pos_is_okay());
 288    }
 289
 290    void delete_lines(size_t cnt)
 291    {
 292       std::list<struct Change>::iterator x = where;
 293       assert(pos_is_okay());
 294       while (cnt > 0)
 295       {
 296          size_t del;
 297          del = x->add_cnt;
 298          if (del > cnt)
 299             del = cnt;
 300          x->skip_lines(del);
 301          cnt -= del;
 302
 303          ++x;
 304          if (x == changes.end()) {
 305             del = cnt;
 306          } else {
 307             del = x->offset;
 308             if (del > cnt)
 309                del = cnt;
 310             x->offset -= del;
 311          }
 312          where->del_cnt += del;
 313          cnt -= del;
 314       }
 315       assert(pos_is_okay());
 316    }
 317
 318    void left(void) {
 319       assert(pos_is_okay());
 320       --where;
 321       pos -= where->offset + where->add_cnt;
 322       assert(pos_is_okay());
 323    }
 324
 325    void right(void) {
 326       assert(pos_is_okay());
 327       pos += where->offset + where->add_cnt;
 328       ++where;
 329       assert(pos_is_okay());
 330    }
 331 };
 332
 333 class Patch {
 334    FileChanges filechanges;
 335    MemBlock add_text;
 336
 337    static bool retry_fwrite(char *b, size_t l, FILE *f, Hashes *hash)
 338    {
 339       size_t r = 1;
 340       while (r > 0 && l > 0)
 341       {
 342          r = fwrite(b, 1, l, f);
 343          if (hash)
 344             hash->Add((unsigned char*)b, r);
 345          l -= r;
 346          b += r;
 347       }
 348       return l == 0;
 349    }
 350
 351    static void dump_rest(FILE *o, FILE *i, Hashes *hash)
 352    {
 353       char buffer[BLOCK_SIZE];
 354       size_t l;
 355       while (0 < (l = fread(buffer, 1, sizeof(buffer), i))) {
 356          if (!retry_fwrite(buffer, l, o, hash))
 357             break;
 358       }
 359    }
 360
 361    static void dump_lines(FILE *o, FILE *i, size_t n, Hashes *hash)
 362    {
 363       char buffer[BLOCK_SIZE];
 364       while (n > 0) {
 365          if (fgets(buffer, sizeof(buffer), i) == 0)
 366             buffer[0] = '\0';
 367          size_t const l = strlen(buffer);
 368          if (l == 0 || buffer[l-1] == '\n')
 369             n--;
 370          retry_fwrite(buffer, l, o, hash);
 371       }
 372    }
 373
 374    static void skip_lines(FILE *i, int n)
 375    {
 376       char buffer[BLOCK_SIZE];
 377       while (n > 0) {
 378          if (fgets(buffer, sizeof(buffer), i) == 0)
 379             buffer[0] = '\0';
 380          size_t const l = strlen(buffer);
 381          if (l == 0 || buffer[l-1] == '\n')
 382             n--;
 383       }
 384    }
 385
 386    static void dump_mem(FILE *o, char *p, size_t s, Hashes *hash) {
 387       retry_fwrite(p, s, o, hash);
 388    }
 389
 390    public:
 391
 392    bool read_diff(FileFd &f, Hashes * const h)
 393    {
 394       char buffer[BLOCK_SIZE];
 395       bool cmdwanted = true;
 396
 397       Change ch(std::numeric_limits<size_t>::max());
 398       if (f.ReadLine(buffer, sizeof(buffer)) == NULL)
 399          return _error->Error("Reading first line of patchfile %s failed", f.Name().c_str());
 400       do {
 401          if (h != NULL)
 402             h->Add(buffer);
 403          if (cmdwanted) {
 404             char *m, *c;
 405             size_t s, e;
 406             errno = 0;
 407             s = strtoul(buffer, &m, 10);
 408             if (unlikely(m == buffer || s == std::numeric_limits<unsigned long>::max() || errno != 0))
 409                return _error->Error("Parsing patchfile %s failed: Expected an effected line start", f.Name().c_str());
 410             else if (*m == ',') {
 411                ++m;
 412                e = strtol(m, &c, 10);
 413                if (unlikely(m == c || e == std::numeric_limits<unsigned long>::max() || errno != 0))
 414                   return _error->Error("Parsing patchfile %s failed: Expected an effected line end", f.Name().c_str());
 415                if (unlikely(e < s))
 416                   return _error->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f.Name().c_str(), e, s);
 417             } else {
 418                e = s;
 419                c = m;
 420             }
 421             if (s > ch.offset)
 422                return _error->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f.Name().c_str());
 423             switch(*c) {
 424                case 'a':
 425                   cmdwanted = false;
 426                   ch.add = NULL;
 427                   ch.add_cnt = 0;
 428                   ch.add_len = 0;
 429                   ch.offset = s;
 430                   ch.del_cnt = 0;
 431                   break;
 432                case 'c':
 433                   if (unlikely(s == 0))
 434                      return _error->Error("Parsing patchfile %s failed: Change command can't effect line zero", f.Name().c_str());
 435                   cmdwanted = false;
 436                   ch.add = NULL;
 437                   ch.add_cnt = 0;
 438                   ch.add_len = 0;
 439                   ch.offset = s - 1;
 440                   ch.del_cnt = e - s + 1;
 441                   break;
 442                case 'd':
 443                   if (unlikely(s == 0))
 444                      return _error->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f.Name().c_str());
 445                   ch.offset = s - 1;
 446                   ch.del_cnt = e - s + 1;
 447                   ch.add = NULL;
 448                   ch.add_cnt = 0;
 449                   ch.add_len = 0;
 450                   filechanges.add_change(ch);
 451                   break;
 452                default:
 453                   return _error->Error("Parsing patchfile %s failed: Unknown command", f.Name().c_str());
 454             }
 455          } else { /* !cmdwanted */
 456             if (strcmp(buffer, ".\n") == 0) {
 457                cmdwanted = true;
 458                filechanges.add_change(ch);
 459             } else {
 460                char *last = NULL;
 461                char *add;
 462                size_t l;
 463                if (ch.add)
 464                   last = ch.add + ch.add_len;
 465                l = strlen(buffer);
 466                add = add_text.add_easy(buffer, l, last);
 467                if (!add) {
 468                   ch.add_len += l;
 469                   ch.add_cnt++;
 470                } else {
 471                   if (ch.add) {
 472                      filechanges.add_change(ch);
 473                      ch.del_cnt = 0;
 474                   }
 475                   ch.offset += ch.add_cnt;
 476                   ch.add = add;
 477                   ch.add_len = l;
 478                   ch.add_cnt = 1;
 479                }
 480             }
 481          }
 482       } while(f.ReadLine(buffer, sizeof(buffer)));
 483       return true;
 484    }
 485
 486    void write_diff(FILE *f)
 487    {
 488       unsigned long long line = 0;
 489       std::list<struct Change>::reverse_iterator ch;
 490       for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) {
 491          line += ch->offset + ch->del_cnt;
 492       }
 493
 494       for (ch = filechanges.rbegin(); ch != filechanges.rend(); ++ch) {
 495          std::list<struct Change>::reverse_iterator mg_i, mg_e = ch;
 496          while (ch->del_cnt == 0 && ch->offset == 0)
 497             ++ch;
 498          line -= ch->del_cnt;
 499          if (ch->add_cnt > 0) {
 500             if (ch->del_cnt == 0) {
 501                fprintf(f, "%llua\n", line);
 502             } else if (ch->del_cnt == 1) {
 503                fprintf(f, "%lluc\n", line+1);
 504             } else {
 505                fprintf(f, "%llu,%lluc\n", line+1, line+ch->del_cnt);
 506             }
 507
 508             mg_i = ch;
 509             do {
 510                dump_mem(f, mg_i->add, mg_i->add_len, NULL);
 511             } while (mg_i-- != mg_e);
 512
 513             fprintf(f, ".\n");
 514          } else if (ch->del_cnt == 1) {
 515             fprintf(f, "%llud\n", line+1);
 516          } else if (ch->del_cnt > 1) {
 517             fprintf(f, "%llu,%llud\n", line+1, line+ch->del_cnt);
 518          }
 519          line -= ch->offset;
 520       }
 521    }
 522
 523    void apply_against_file(FILE *out, FILE *in, Hashes *hash = NULL)
 524    {
 525       std::list<struct Change>::iterator ch;
 526       for (ch = filechanges.begin(); ch != filechanges.end(); ++ch) {
 527          dump_lines(out, in, ch->offset, hash);
 528          skip_lines(in, ch->del_cnt);
 529          dump_mem(out, ch->add, ch->add_len, hash);
 530       }
 531       dump_rest(out, in, hash);
 532    }
 533 };
 534
 535 class RredMethod : public pkgAcqMethod {
 536    private:
 537       bool Debug;
 538
 539       struct PDiffFile {
 540          std::string FileName;
 541          HashStringList ExpectedHashes;
 542          PDiffFile(std::string const &FileName, HashStringList const &ExpectedHashes) :
 543             FileName(FileName), ExpectedHashes(ExpectedHashes) {}
 544       };
 545
 546       HashStringList ReadExpectedHashesForPatch(unsigned int const patch, std::string const &Message)
 547       {
 548          HashStringList ExpectedHashes;
 549          for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 550          {
 551             std::string tagname;
 552             strprintf(tagname, "Patch-%d-%s-Hash", patch, *type);
 553             std::string const hashsum = LookupTag(Message, tagname.c_str());
 554             if (hashsum.empty() == false)
 555                ExpectedHashes.push_back(HashString(*type, hashsum));
 556          }
 557          return ExpectedHashes;
 558       }
 559
 560    protected:
 561       virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE {
 562          Debug = _config->FindB("Debug::pkgAcquire::RRed", false);
 563          URI Get = Itm->Uri;
 564          std::string Path = Get.Host + Get.Path; // rred:/path - no host
 565
 566          FetchResult Res;
 567          Res.Filename = Itm->DestFile;
 568          if (Itm->Uri.empty())
 569          {
 570             Path = Itm->DestFile;
 571             Itm->DestFile.append(".result");
 572          } else
 573             URIStart(Res);
 574
 575          std::vector<PDiffFile> patchfiles;
 576          Patch patch;
 577
 578          if (FileExists(Path + ".ed") == true)
 579          {
 580             HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(0, Message);
 581             std::string const FileName = Path + ".ed";
 582             if (ExpectedHashes.usable() == false)
 583                return _error->Error("No hashes found for uncompressed patch: %s", FileName.c_str());
 584             patchfiles.push_back(PDiffFile(FileName, ExpectedHashes));
 585          }
 586          else
 587          {
 588             _error->PushToStack();
 589             std::vector<std::string> patches = GetListOfFilesInDir(flNotFile(Path), "gz", true, false);
 590             _error->RevertToStack();
 591
 592             std::string const baseName = Path + ".ed.";
 593             unsigned int seen_patches = 0;
 594             for (std::vector<std::string>::const_iterator p = patches.begin();
 595                   p != patches.end(); ++p)
 596             {
 597                if (p->compare(0, baseName.length(), baseName) == 0)
 598                {
 599                   HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(seen_patches, Message);
 600                   if (ExpectedHashes.usable() == false)
 601                      return _error->Error("No hashes found for uncompressed patch %d: %s", seen_patches, p->c_str());
 602                   patchfiles.push_back(PDiffFile(*p, ExpectedHashes));
 603                   ++seen_patches;
 604                }
 605             }
 606          }
 607
 608          std::string patch_name;
 609          for (std::vector<PDiffFile>::iterator I = patchfiles.begin();
 610                I != patchfiles.end();
 611                ++I)
 612          {
 613             patch_name = I->FileName;
 614             if (Debug == true)
 615                std::clog << "Patching " << Path << " with " << patch_name
 616                   << std::endl;
 617
 618             FileFd p;
 619             Hashes patch_hash(I->ExpectedHashes);
 620             // all patches are compressed, even if the name doesn't reflect it
 621             if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false ||
 622                   patch.read_diff(p, &patch_hash) == false)
 623             {
 624                _error->DumpErrors(std::cerr);
 625                return false;
 626             }
 627             p.Close();
 628             HashStringList const hsl = patch_hash.GetHashStringList();
 629             if (hsl != I->ExpectedHashes)
 630                return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
 631          }
 632
 633          if (Debug == true)
 634             std::clog << "Applying patches against " << Path
 635                << " and writing results to " << Itm->DestFile
 636                << std::endl;
 637
 638          FILE *inp = fopen(Path.c_str(), "r");
 639          FILE *out = fopen(Itm->DestFile.c_str(), "w");
 640
 641          Hashes hash(Itm->ExpectedHashes);
 642          patch.apply_against_file(out, inp, &hash);
 643
 644          fclose(out);
 645          fclose(inp);
 646
 647          if (Debug == true) {
 648             std::clog << "rred: finished file patching of " << Path  << "." << std::endl;
 649          }
 650
 651          struct stat bufbase, bufpatch;
 652          if (stat(Path.c_str(), &bufbase) != 0 ||
 653                stat(patch_name.c_str(), &bufpatch) != 0)
 654             return _error->Errno("stat", _("Failed to stat"));
 655
 656          struct timeval times[2];
 657          times[0].tv_sec = bufbase.st_atime;
 658          times[1].tv_sec = bufpatch.st_mtime;
 659          times[0].tv_usec = times[1].tv_usec = 0;
 660          if (utimes(Itm->DestFile.c_str(), times) != 0)
 661             return _error->Errno("utimes",_("Failed to set modification time"));
 662
 663          if (stat(Itm->DestFile.c_str(), &bufbase) != 0)
 664             return _error->Errno("stat", _("Failed to stat"));
 665
 666          Res.LastModified = bufbase.st_mtime;
 667          Res.Size = bufbase.st_size;
 668          Res.TakeHashes(hash);
 669          URIDone(Res);
 670
 671          return true;
 672       }
 673
 674       bool Configuration(std::string Message) APT_OVERRIDE
 675       {
 676          if (pkgAcqMethod::Configuration(Message) == false)
 677             return false;
 678
 679          DropPrivsOrDie();
 680
 681          return true;
 682       }
 683
 684    public:
 685       RredMethod() : pkgAcqMethod("2.0",SingleInstance | SendConfig), Debug(false) {}
 686 };
 687
 688 int main(int argc, char **argv)
 689 {
 690    int i;
 691    bool just_diff = true;
 692    Patch patch;
 693
 694    if (argc <= 1) {
 695       RredMethod Mth;
 696       return Mth.Run();
 697    }
 698
 699    if (argc > 1 && strcmp(argv[1], "-f") == 0) {
 700       just_diff = false;
 701       i = 2;
 702    } else {
 703       i = 1;
 704    }
 705
 706    for (; i < argc; i++) {
 707       FileFd p;
 708       if (p.Open(argv[i], FileFd::ReadOnly) == false) {
 709          _error->DumpErrors(std::cerr);
 710          exit(1);
 711       }
 712       if (patch.read_diff(p, NULL) == false)
 713       {
 714          _error->DumpErrors(std::cerr);
 715          exit(2);
 716       }
 717    }
 718
 719    if (just_diff) {
 720       patch.write_diff(stdout);
 721    } else {
 722       FILE *out, *inp;
 723       out = stdout;
 724       inp = stdin;
 725
 726       patch.apply_against_file(out, inp);
 727    }
 728    return 0;
 729 }