apt-pkg/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.10 2002/03/26 07:38:58 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/indexrecords.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/cdrom.h>
  24 #include <apti18n.h>
  25
  26 #include <iostream>
  27 #include <sstream>
  28 #include <unistd.h>
  29 #include <sys/stat.h>
  30 #include <stdio.h>
  31                                                                         /*}}}*/
  32
  33 using namespace std;
  34
  35 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  36 // ---------------------------------------------------------------------
  37 /* */
  38 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
  39                              pkgCdromStatus *log)
  40 {
  41    OpProgress *Progress = NULL;
  42    if (List.size() == 0)
  43       return true;
  44
  45    if(log)
  46       Progress = log->GetOpProgress();
  47
  48    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  49    bool Debug = _config->FindB("Debug::aptcdrom",false);
  50
  51    // Prepare the progress indicator
  52    unsigned long TotalSize = 0;
  53    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  54    {
  55       struct stat Buf;
  56       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  57           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  58          return _error->Errno("stat","Stat failed for %s",
  59                               string(*I + GetFileName()).c_str());
  60       TotalSize += Buf.st_size;
  61    }
  62
  63    unsigned long CurrentSize = 0;
  64    unsigned int NotFound = 0;
  65    unsigned int WrongSize = 0;
  66    unsigned int Packages = 0;
  67    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  68    {
  69       string OrigPath = string(*I,CDROM.length());
  70       unsigned long FileSize = 0;
  71
  72       // Open the package file
  73       FileFd Pkg;
  74       if (FileExists(*I + GetFileName()) == true)
  75       {
  76          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  77          FileSize = Pkg.Size();
  78       }
  79       else
  80       {
  81          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  82          if (_error->PendingError() == true)
  83             return false;
  84          FileSize = From.Size();
  85
  86          // Get a temp file
  87          FILE *tmp = tmpfile();
  88          if (tmp == 0)
  89             return _error->Errno("tmpfile","Unable to create a tmp file");
  90          Pkg.Fd(dup(fileno(tmp)));
  91          fclose(tmp);
  92
  93          // Fork gzip
  94          pid_t Process = fork();
  95          if (Process < 0)
  96             return _error->Errno("fork","Couldn't fork gzip");
  97
  98          // The child
  99          if (Process == 0)
 100          {
 101             dup2(From.Fd(),STDIN_FILENO);
 102             dup2(Pkg.Fd(),STDOUT_FILENO);
 103             SetCloseExec(STDIN_FILENO,false);
 104             SetCloseExec(STDOUT_FILENO,false);
 105
 106             const char *Args[3];
 107             string Tmp =  _config->Find("Dir::bin::gzip","gzip");
 108             Args[0] = Tmp.c_str();
 109             Args[1] = "-d";
 110             Args[2] = 0;
 111             execvp(Args[0],(char **)Args);
 112             exit(100);
 113          }
 114
 115          // Wait for gzip to finish
 116          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 117             return _error->Error("gzip failed, perhaps the disk is full.");
 118
 119          Pkg.Seek(0);
 120       }
 121       pkgTagFile Parser(&Pkg);
 122       if (_error->PendingError() == true)
 123          return false;
 124
 125       // Open the output file
 126       char S[400];
 127       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",Name.c_str(),
 128                (*I).c_str() + CDROM.length(),GetFileName());
 129       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 130       TargetF += URItoFileName(S);
 131       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 132          TargetF = "/dev/null";
 133       FileFd Target(TargetF,FileFd::WriteEmpty);
 134       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 135       if (_error->PendingError() == true)
 136          return false;
 137       if (TargetFl == 0)
 138          return _error->Errno("fdopen","Failed to reopen fd");
 139
 140       // Setup the progress meter
 141       if(Progress)
 142          Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
 143                                    string("Reading ") + Type() + " Indexes");
 144
 145       // Parse
 146       if(Progress)
 147          Progress->SubProgress(Pkg.Size());
 148       pkgTagSection Section;
 149       this->Section = &Section;
 150       string Prefix;
 151       unsigned long Hits = 0;
 152       unsigned long Chop = 0;
 153       while (Parser.Step(Section) == true)
 154       {
 155          if(Progress)
 156             Progress->Progress(Parser.Offset());
 157          string File;
 158          unsigned long Size;
 159          if (GetFile(File,Size) == false)
 160          {
 161             fclose(TargetFl);
 162             return false;
 163          }
 164
 165          if (Chop != 0)
 166             File = OrigPath + ChopDirs(File,Chop);
 167
 168          // See if the file exists
 169          bool Mangled = false;
 170          if (NoStat == false || Hits < 10)
 171          {
 172             // Attempt to fix broken structure
 173             if (Hits == 0)
 174             {
 175                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 176                    ReconstructChop(Chop,*I,File) == false)
 177                {
 178                   if (Debug == true)
 179                      clog << "Missed: " << File << endl;
 180                   NotFound++;
 181                   continue;
 182                }
 183                if (Chop != 0)
 184                   File = OrigPath + ChopDirs(File,Chop);
 185             }
 186
 187             // Get the size
 188             struct stat Buf;
 189             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 190                 Buf.st_size == 0)
 191             {
 192                // Attempt to fix busted symlink support for one instance
 193                string OrigFile = File;
 194                string::size_type Start = File.find("binary-");
 195                string::size_type End = File.find("/",Start+3);
 196                if (Start != string::npos && End != string::npos)
 197                {
 198                   File.replace(Start,End-Start,"binary-all");
 199                   Mangled = true;
 200                }
 201
 202                if (Mangled == false ||
 203                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 204                {
 205                   if (Debug == true)
 206                      clog << "Missed(2): " << OrigFile << endl;
 207                   NotFound++;
 208                   continue;
 209                }
 210             }
 211
 212             // Size match
 213             if ((unsigned)Buf.st_size != Size)
 214             {
 215                if (Debug == true)
 216                   clog << "Wrong Size: " << File << endl;
 217                WrongSize++;
 218                continue;
 219             }
 220          }
 221
 222          Packages++;
 223          Hits++;
 224
 225          if (RewriteEntry(TargetFl,File) == false)
 226          {
 227             fclose(TargetFl);
 228             return false;
 229          }
 230       }
 231       fclose(TargetFl);
 232
 233       if (Debug == true)
 234          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 235
 236       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 237       {
 238          // Move out of the partial directory
 239          Target.Close();
 240          string FinalF = _config->FindDir("Dir::State::lists");
 241          FinalF += URItoFileName(S);
 242          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 243             return _error->Errno("rename","Failed to rename");
 244       }
 245
 246       /* Mangle the source to be in the proper notation with
 247          prefix dist [component] */
 248       *I = string(*I,Prefix.length());
 249       ConvertToSourceList(CDROM,*I);
 250       *I = Prefix + ' ' + *I;
 251
 252       CurrentSize += FileSize;
 253    }
 254    if(Progress)
 255       Progress->Done();
 256
 257    // Some stats
 258    if(log) {
 259       stringstream msg;
 260       if(NotFound == 0 && WrongSize == 0)
 261          ioprintf(msg, _("Wrote %i records.\n"), Packages);
 262       else if (NotFound != 0 && WrongSize == 0)
 263          ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 264                   Packages, NotFound);
 265       else if (NotFound == 0 && WrongSize != 0)
 266          ioprintf(msg, _("Wrote %i records with %i mismachted files\n"),
 267                   Packages, WrongSize);
 268       if (NotFound != 0 && WrongSize != 0)
 269          ioprintf(msg, _("Wrote %i records with %i missing files and %i mismachted files\n"), Packages, NotFound, WrongSize);
 270    }
 271
 272    if (Packages == 0)
 273       _error->Warning("No valid records were found.");
 274
 275    if (NotFound + WrongSize > 10)
 276       _error->Warning("Alot of entries were discarded, something may be wrong.\n");
 277
 278
 279    return true;
 280 }
 281                                                                         /*}}}*/
 282 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 283 // ---------------------------------------------------------------------
 284 /* */
 285 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 286 {
 287    string::size_type I = 0;
 288    do
 289    {
 290       I = Path.find('/',I+1);
 291       Depth--;
 292    }
 293    while (I != string::npos && Depth != 0);
 294
 295    if (I == string::npos)
 296       return string();
 297
 298    return string(Path,I+1);
 299 }
 300                                                                         /*}}}*/
 301 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 302 // ---------------------------------------------------------------------
 303 /* This prepends dir components from the path to the package files to
 304    the path to the deb until it is found */
 305 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 306                                   string File)
 307 {
 308    bool Debug = _config->FindB("Debug::aptcdrom",false);
 309    unsigned int Depth = 1;
 310    string MyPrefix = Prefix;
 311    while (1)
 312    {
 313       struct stat Buf;
 314       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 315       {
 316          if (Debug == true)
 317             cout << "Failed, " << CD + MyPrefix + File << endl;
 318          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 319             continue;
 320
 321          return false;
 322       }
 323       else
 324       {
 325          Prefix = MyPrefix;
 326          return true;
 327       }
 328    }
 329    return false;
 330 }
 331                                                                         /*}}}*/
 332 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 333 // ---------------------------------------------------------------------
 334 /* This removes path components from the filename and prepends the location
 335    of the package files until a file is found */
 336 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 337 {
 338    // Attempt to reconstruct the filename
 339    unsigned long Depth = 0;
 340    while (1)
 341    {
 342       struct stat Buf;
 343       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 344       {
 345          File = ChopDirs(File,1);
 346          Depth++;
 347          if (File.empty() == false)
 348             continue;
 349          return false;
 350       }
 351       else
 352       {
 353          Chop = Depth;
 354          return true;
 355       }
 356    }
 357    return false;
 358 }
 359                                                                         /*}}}*/
 360 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 361 // ---------------------------------------------------------------------
 362 /* We look for things in dists/ notation and convert them to
 363    <dist> <component> form otherwise it is left alone. This also strips
 364    the CD path.
 365
 366    This implements a regex sort of like:
 367     (.*)/dists/([^/]*)/(.*)/binary-*
 368      ^          ^      ^- Component
 369      |          |-------- Distribution
 370      |------------------- Path
 371
 372    It was deciced to use only a single word for dist (rather than say
 373    unstable/non-us) to increase the chance that each CD gets a single
 374    line in sources.list.
 375  */
 376 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 377 {
 378    char S[300];
 379    snprintf(S,sizeof(S),"binary-%s",_config->Find("Apt::Architecture").c_str());
 380
 381    // Strip the cdrom base path
 382    Path = string(Path,CD.length());
 383    if (Path.empty() == true)
 384       Path = "/";
 385
 386    // Too short to be a dists/ type
 387    if (Path.length() < strlen("dists/"))
 388       return;
 389
 390    // Not a dists type.
 391    if (stringcmp(Path.c_str(),Path.c_str()+strlen("dists/"),"dists/") != 0)
 392       return;
 393
 394    // Isolate the dist
 395    string::size_type Slash = strlen("dists/");
 396    string::size_type Slash2 = Path.find('/',Slash + 1);
 397    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 398       return;
 399    string Dist = string(Path,Slash,Slash2 - Slash);
 400
 401    // Isolate the component
 402    Slash = Slash2;
 403    for (unsigned I = 0; I != 10; I++)
 404    {
 405       Slash = Path.find('/',Slash+1);
 406       if (Slash == string::npos || Slash + 2 >= Path.length())
 407          return;
 408       string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 409
 410       // Verify the trailing binary- bit
 411       string::size_type BinSlash = Path.find('/',Slash + 1);
 412       if (Slash == string::npos)
 413          return;
 414       string Binary = string(Path,Slash+1,BinSlash - Slash-1);
 415
 416       if (Binary != S && Binary != "source")
 417          continue;
 418
 419       Path = Dist + ' ' + Comp;
 420       return;
 421    }
 422 }
 423                                                                         /*}}}*/
 424 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 425 // ---------------------------------------------------------------------
 426 /* */
 427 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 428 {
 429    string::size_type I = 0;
 430    do
 431    {
 432       I = Path.find('/',I+1);
 433       Depth--;
 434    }
 435    while (I != string::npos && Depth != 0);
 436
 437    if (I == string::npos)
 438       return false;
 439
 440    To = string(Path,0,I+1);
 441    return true;
 442 }
 443                                                                         /*}}}*/
 444 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 445 // ---------------------------------------------------------------------
 446 /* */
 447 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 448 {
 449    File = Section->FindS("Filename");
 450    Size = Section->FindI("Size");
 451    if (File.empty() || Size == 0)
 452       return _error->Error("Cannot find filename or size tag");
 453    return true;
 454 }
 455                                                                         /*}}}*/
 456 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 457 // ---------------------------------------------------------------------
 458 /* */
 459 bool PackageCopy::RewriteEntry(FILE *Target,string File)
 460 {
 461    TFRewriteData Changes[] = {{"Filename",File.c_str()},
 462                               {}};
 463
 464    if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
 465       return false;
 466    fputc('\n',Target);
 467    return true;
 468 }
 469                                                                         /*}}}*/
 470 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 471 // ---------------------------------------------------------------------
 472 /* */
 473 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 474 {
 475    string Files = Section->FindS("Files");
 476    if (Files.empty() == true)
 477       return false;
 478
 479    // Stash the / terminated directory prefix
 480    string Base = Section->FindS("Directory");
 481    if (Base.empty() == false && Base[Base.length()-1] != '/')
 482       Base += '/';
 483
 484    // Read the first file triplet
 485    const char *C = Files.c_str();
 486    string sSize;
 487    string MD5Hash;
 488
 489    // Parse each of the elements
 490    if (ParseQuoteWord(C,MD5Hash) == false ||
 491        ParseQuoteWord(C,sSize) == false ||
 492        ParseQuoteWord(C,File) == false)
 493       return _error->Error("Error parsing file record");
 494
 495    // Parse the size and append the directory
 496    Size = atoi(sSize.c_str());
 497    File = Base + File;
 498    return true;
 499 }
 500                                                                         /*}}}*/
 501 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 502 // ---------------------------------------------------------------------
 503 /* */
 504 bool SourceCopy::RewriteEntry(FILE *Target,string File)
 505 {
 506    string Dir(File,0,File.rfind('/'));
 507    TFRewriteData Changes[] = {{"Directory",Dir.c_str()},
 508                               {}};
 509
 510    if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
 511       return false;
 512    fputc('\n',Target);
 513    return true;
 514 }
 515
 516
 517                                                                         /*}}}*/
 518
 519 bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex)
 520 {
 521    const indexRecords::checkSum *Record = MetaIndex->Lookup(file);
 522
 523    if (!Record)
 524    {
 525       _error->Warning("Can't find authentication record for: %s",file.c_str());
 526       return false;
 527    }
 528
 529    MD5Summation sum;
 530    FileFd Fd(prefix+file, FileFd::ReadOnly);
 531    sum.AddFD(Fd.Fd(), Fd.Size());
 532    Fd.Close();
 533    string MD5 = (string)sum.Result();
 534
 535    if (Record->MD5Hash != MD5)
 536    {
 537       _error->Warning("MD5 mismatch for: %s",file.c_str());
 538       return false;
 539    }
 540
 541    if(_config->FindB("Debug::aptcdrom",false))
 542    {
 543       cout << "File: " << prefix+file << endl;
 544       cout << "Expected MD5sum: " << Record->MD5Hash << endl;
 545       cout << "got: " << MD5 << endl << endl;
 546    }
 547
 548    return true;
 549 }
 550
 551 bool SigVerify::CopyMetaIndex(string CDROM, string CDName,
 552                               string prefix, string file)
 553 {
 554       char S[400];
 555       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",CDName.c_str(),
 556                (prefix).c_str() + CDROM.length(),file.c_str());
 557       string TargetF = _config->FindDir("Dir::State::lists");
 558       TargetF += URItoFileName(S);
 559
 560       FileFd Target;
 561       FileFd Rel;
 562       Target.Open(TargetF,FileFd::WriteEmpty);
 563       Rel.Open(prefix + file,FileFd::ReadOnly);
 564       if (_error->PendingError() == true)
 565          return false;
 566       if (CopyFile(Rel,Target) == false)
 567          return false;
 568
 569       return true;
 570 }
 571
 572 bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList,
 573                               vector<string> PkgList,vector<string> SrcList)
 574 {
 575    if (SigList.size() == 0)
 576       return true;
 577
 578    bool Debug = _config->FindB("Debug::aptcdrom",false);
 579
 580    // Read all Release files
 581    for (vector<string>::iterator I = SigList.begin(); I != SigList.end(); I++)
 582    {
 583       if(Debug)
 584          cout << "Signature verify for: " << *I << endl;
 585
 586       indexRecords *MetaIndex = new indexRecords;
 587       string prefix = *I;
 588
 589       // a Release.gpg without a Release should never happen
 590       if(!FileExists(*I+"Release"))
 591          continue;
 592
 593
 594       // verify the gpg signature of "Release"
 595       // gpg --verify "*I+Release.gpg", "*I+Release"
 596       string gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv");
 597       string pubringpath = _config->Find("Apt::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg");
 598       pid_t pid = ExecFork();
 599       if(pid < 0) {
 600          _error->Error("Fork failed");
 601          return false;
 602       }
 603       if(pid == 0) {
 604          execlp(gpgvpath.c_str(), gpgvpath.c_str(), "--keyring",
 605                 pubringpath.c_str(), string(*I+"Release.gpg").c_str(),
 606                 string(*I+"Release").c_str(), NULL);
 607       }
 608       if(!ExecWait(pid, "gpgv")) {
 609          _error->Warning("Signature verification failed for: %s",
 610                          string(*I+"Release.gpg").c_str());
 611          // something went wrong, don't copy the Release.gpg
 612          // FIXME: delete any existing gpg file?
 613          continue;
 614       }
 615
 616       // Open the Release file and add it to the MetaIndex
 617       if(!MetaIndex->Load(*I+"Release"))
 618       {
 619          _error->Error(MetaIndex->ErrorText.c_str());
 620          return false;
 621       }
 622
 623       // go over the Indexfiles and see if they verify
 624       // if so, remove them from our copy of the lists
 625       vector<string> keys = MetaIndex->MetaKeys();
 626       for (vector<string>::iterator I = keys.begin(); I != keys.end(); I++)
 627       {
 628          if(!Verify(prefix,*I, MetaIndex)) {
 629             // something went wrong, don't copy the Release.gpg
 630             // FIXME: delete any existing gpg file?
 631             continue;
 632          }
 633       }
 634
 635       // we need a fresh one for the Release.gpg
 636       delete MetaIndex;
 637
 638       // everything was fine, copy the Release and Release.gpg file
 639       CopyMetaIndex(CDROM, Name, prefix, "Release");
 640       CopyMetaIndex(CDROM, Name, prefix, "Release.gpg");
 641    }
 642
 643    return true;
 644 }