apt-pkg/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.10 2002/03/26 07:38:58 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/indexrecords.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/cdrom.h>
  24 #include <apti18n.h>
  25
  26 #include <iostream>
  27 #include <sstream>
  28 #include <unistd.h>
  29 #include <sys/stat.h>
  30 #include <stdio.h>
  31                                                                         /*}}}*/
  32
  33 using namespace std;
  34
  35
  36
  37 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  38 // ---------------------------------------------------------------------
  39 /* */
  40 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
  41                              pkgCdromStatus *log)
  42 {
  43    OpProgress *Progress = NULL;
  44    if (List.size() == 0)
  45       return true;
  46
  47    if(log)
  48       Progress = log->GetOpProgress();
  49
  50    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  51    bool Debug = _config->FindB("Debug::aptcdrom",false);
  52
  53    // Prepare the progress indicator
  54    unsigned long TotalSize = 0;
  55    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  56    {
  57       struct stat Buf;
  58       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  59           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  60          return _error->Errno("stat","Stat failed for %s",
  61                               string(*I + GetFileName()).c_str());
  62       TotalSize += Buf.st_size;
  63    }
  64
  65    unsigned long CurrentSize = 0;
  66    unsigned int NotFound = 0;
  67    unsigned int WrongSize = 0;
  68    unsigned int Packages = 0;
  69    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  70    {
  71       string OrigPath = string(*I,CDROM.length());
  72       unsigned long FileSize = 0;
  73
  74       // Open the package file
  75       FileFd Pkg;
  76       if (FileExists(*I + GetFileName()) == true)
  77       {
  78          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  79          FileSize = Pkg.Size();
  80       }
  81       else
  82       {
  83          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  84          if (_error->PendingError() == true)
  85             return false;
  86          FileSize = From.Size();
  87
  88          // Get a temp file
  89          FILE *tmp = tmpfile();
  90          if (tmp == 0)
  91             return _error->Errno("tmpfile","Unable to create a tmp file");
  92          Pkg.Fd(dup(fileno(tmp)));
  93          fclose(tmp);
  94
  95          // Fork gzip
  96          pid_t Process = fork();
  97          if (Process < 0)
  98             return _error->Errno("fork","Couldn't fork gzip");
  99
 100          // The child
 101          if (Process == 0)
 102          {
 103             dup2(From.Fd(),STDIN_FILENO);
 104             dup2(Pkg.Fd(),STDOUT_FILENO);
 105             SetCloseExec(STDIN_FILENO,false);
 106             SetCloseExec(STDOUT_FILENO,false);
 107
 108             const char *Args[3];
 109             string Tmp =  _config->Find("Dir::bin::gzip","gzip");
 110             Args[0] = Tmp.c_str();
 111             Args[1] = "-d";
 112             Args[2] = 0;
 113             execvp(Args[0],(char **)Args);
 114             exit(100);
 115          }
 116
 117          // Wait for gzip to finish
 118          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 119             return _error->Error("gzip failed, perhaps the disk is full.");
 120
 121          Pkg.Seek(0);
 122       }
 123       pkgTagFile Parser(&Pkg);
 124       if (_error->PendingError() == true)
 125          return false;
 126
 127       // Open the output file
 128       char S[400];
 129       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",Name.c_str(),
 130                (*I).c_str() + CDROM.length(),GetFileName());
 131       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 132       TargetF += URItoFileName(S);
 133       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 134          TargetF = "/dev/null";
 135       FileFd Target(TargetF,FileFd::WriteEmpty);
 136       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 137       if (_error->PendingError() == true)
 138          return false;
 139       if (TargetFl == 0)
 140          return _error->Errno("fdopen","Failed to reopen fd");
 141
 142       // Setup the progress meter
 143       if(Progress)
 144          Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
 145                                    string("Reading ") + Type() + " Indexes");
 146
 147       // Parse
 148       if(Progress)
 149          Progress->SubProgress(Pkg.Size());
 150       pkgTagSection Section;
 151       this->Section = &Section;
 152       string Prefix;
 153       unsigned long Hits = 0;
 154       unsigned long Chop = 0;
 155       while (Parser.Step(Section) == true)
 156       {
 157          if(Progress)
 158             Progress->Progress(Parser.Offset());
 159          string File;
 160          unsigned long Size;
 161          if (GetFile(File,Size) == false)
 162          {
 163             fclose(TargetFl);
 164             return false;
 165          }
 166
 167          if (Chop != 0)
 168             File = OrigPath + ChopDirs(File,Chop);
 169
 170          // See if the file exists
 171          bool Mangled = false;
 172          if (NoStat == false || Hits < 10)
 173          {
 174             // Attempt to fix broken structure
 175             if (Hits == 0)
 176             {
 177                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 178                    ReconstructChop(Chop,*I,File) == false)
 179                {
 180                   if (Debug == true)
 181                      clog << "Missed: " << File << endl;
 182                   NotFound++;
 183                   continue;
 184                }
 185                if (Chop != 0)
 186                   File = OrigPath + ChopDirs(File,Chop);
 187             }
 188
 189             // Get the size
 190             struct stat Buf;
 191             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 192                 Buf.st_size == 0)
 193             {
 194                // Attempt to fix busted symlink support for one instance
 195                string OrigFile = File;
 196                string::size_type Start = File.find("binary-");
 197                string::size_type End = File.find("/",Start+3);
 198                if (Start != string::npos && End != string::npos)
 199                {
 200                   File.replace(Start,End-Start,"binary-all");
 201                   Mangled = true;
 202                }
 203
 204                if (Mangled == false ||
 205                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 206                {
 207                   if (Debug == true)
 208                      clog << "Missed(2): " << OrigFile << endl;
 209                   NotFound++;
 210                   continue;
 211                }
 212             }
 213
 214             // Size match
 215             if ((unsigned)Buf.st_size != Size)
 216             {
 217                if (Debug == true)
 218                   clog << "Wrong Size: " << File << endl;
 219                WrongSize++;
 220                continue;
 221             }
 222          }
 223
 224          Packages++;
 225          Hits++;
 226
 227          if (RewriteEntry(TargetFl,File) == false)
 228          {
 229             fclose(TargetFl);
 230             return false;
 231          }
 232       }
 233       fclose(TargetFl);
 234
 235       if (Debug == true)
 236          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 237
 238       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 239       {
 240          // Move out of the partial directory
 241          Target.Close();
 242          string FinalF = _config->FindDir("Dir::State::lists");
 243          FinalF += URItoFileName(S);
 244          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 245             return _error->Errno("rename","Failed to rename");
 246       }
 247
 248       /* Mangle the source to be in the proper notation with
 249          prefix dist [component] */
 250       *I = string(*I,Prefix.length());
 251       ConvertToSourceList(CDROM,*I);
 252       *I = Prefix + ' ' + *I;
 253
 254       CurrentSize += FileSize;
 255    }
 256    if(Progress)
 257       Progress->Done();
 258
 259    // Some stats
 260    if(log) {
 261       stringstream msg;
 262       if(NotFound == 0 && WrongSize == 0)
 263          ioprintf(msg, _("Wrote %i records.\n"), Packages);
 264       else if (NotFound != 0 && WrongSize == 0)
 265          ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 266                   Packages, NotFound);
 267       else if (NotFound == 0 && WrongSize != 0)
 268          ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 269                   Packages, WrongSize);
 270       if (NotFound != 0 && WrongSize != 0)
 271          ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
 272    }
 273
 274    if (Packages == 0)
 275       _error->Warning("No valid records were found.");
 276
 277    if (NotFound + WrongSize > 10)
 278       _error->Warning("Alot of entries were discarded, something may be wrong.\n");
 279
 280
 281    return true;
 282 }
 283                                                                         /*}}}*/
 284 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 285 // ---------------------------------------------------------------------
 286 /* */
 287 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 288 {
 289    string::size_type I = 0;
 290    do
 291    {
 292       I = Path.find('/',I+1);
 293       Depth--;
 294    }
 295    while (I != string::npos && Depth != 0);
 296
 297    if (I == string::npos)
 298       return string();
 299
 300    return string(Path,I+1);
 301 }
 302                                                                         /*}}}*/
 303 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 304 // ---------------------------------------------------------------------
 305 /* This prepends dir components from the path to the package files to
 306    the path to the deb until it is found */
 307 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 308                                   string File)
 309 {
 310    bool Debug = _config->FindB("Debug::aptcdrom",false);
 311    unsigned int Depth = 1;
 312    string MyPrefix = Prefix;
 313    while (1)
 314    {
 315       struct stat Buf;
 316       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 317       {
 318          if (Debug == true)
 319             cout << "Failed, " << CD + MyPrefix + File << endl;
 320          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 321             continue;
 322
 323          return false;
 324       }
 325       else
 326       {
 327          Prefix = MyPrefix;
 328          return true;
 329       }
 330    }
 331    return false;
 332 }
 333                                                                         /*}}}*/
 334 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 335 // ---------------------------------------------------------------------
 336 /* This removes path components from the filename and prepends the location
 337    of the package files until a file is found */
 338 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 339 {
 340    // Attempt to reconstruct the filename
 341    unsigned long Depth = 0;
 342    while (1)
 343    {
 344       struct stat Buf;
 345       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 346       {
 347          File = ChopDirs(File,1);
 348          Depth++;
 349          if (File.empty() == false)
 350             continue;
 351          return false;
 352       }
 353       else
 354       {
 355          Chop = Depth;
 356          return true;
 357       }
 358    }
 359    return false;
 360 }
 361                                                                         /*}}}*/
 362 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 363 // ---------------------------------------------------------------------
 364 /* We look for things in dists/ notation and convert them to
 365    <dist> <component> form otherwise it is left alone. This also strips
 366    the CD path.
 367
 368    This implements a regex sort of like:
 369     (.*)/dists/([^/]*)/(.*)/binary-*
 370      ^          ^      ^- Component
 371      |          |-------- Distribution
 372      |------------------- Path
 373
 374    It was deciced to use only a single word for dist (rather than say
 375    unstable/non-us) to increase the chance that each CD gets a single
 376    line in sources.list.
 377  */
 378 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 379 {
 380    char S[300];
 381    snprintf(S,sizeof(S),"binary-%s",_config->Find("Apt::Architecture").c_str());
 382
 383    // Strip the cdrom base path
 384    Path = string(Path,CD.length());
 385    if (Path.empty() == true)
 386       Path = "/";
 387
 388    // Too short to be a dists/ type
 389    if (Path.length() < strlen("dists/"))
 390       return;
 391
 392    // Not a dists type.
 393    if (stringcmp(Path.c_str(),Path.c_str()+strlen("dists/"),"dists/") != 0)
 394       return;
 395
 396    // Isolate the dist
 397    string::size_type Slash = strlen("dists/");
 398    string::size_type Slash2 = Path.find('/',Slash + 1);
 399    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 400       return;
 401    string Dist = string(Path,Slash,Slash2 - Slash);
 402
 403    // Isolate the component
 404    Slash = Slash2;
 405    for (unsigned I = 0; I != 10; I++)
 406    {
 407       Slash = Path.find('/',Slash+1);
 408       if (Slash == string::npos || Slash + 2 >= Path.length())
 409          return;
 410       string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 411
 412       // Verify the trailing binary- bit
 413       string::size_type BinSlash = Path.find('/',Slash + 1);
 414       if (Slash == string::npos)
 415          return;
 416       string Binary = string(Path,Slash+1,BinSlash - Slash-1);
 417
 418       if (Binary != S && Binary != "source")
 419          continue;
 420
 421       Path = Dist + ' ' + Comp;
 422       return;
 423    }
 424 }
 425                                                                         /*}}}*/
 426 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 427 // ---------------------------------------------------------------------
 428 /* */
 429 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 430 {
 431    string::size_type I = 0;
 432    do
 433    {
 434       I = Path.find('/',I+1);
 435       Depth--;
 436    }
 437    while (I != string::npos && Depth != 0);
 438
 439    if (I == string::npos)
 440       return false;
 441
 442    To = string(Path,0,I+1);
 443    return true;
 444 }
 445                                                                         /*}}}*/
 446 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* */
 449 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 450 {
 451    File = Section->FindS("Filename");
 452    Size = Section->FindI("Size");
 453    if (File.empty() || Size == 0)
 454       return _error->Error("Cannot find filename or size tag");
 455    return true;
 456 }
 457                                                                         /*}}}*/
 458 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* */
 461 bool PackageCopy::RewriteEntry(FILE *Target,string File)
 462 {
 463    TFRewriteData Changes[] = {{"Filename",File.c_str()},
 464                               {}};
 465
 466    if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
 467       return false;
 468    fputc('\n',Target);
 469    return true;
 470 }
 471                                                                         /*}}}*/
 472 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* */
 475 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 476 {
 477    string Files = Section->FindS("Files");
 478    if (Files.empty() == true)
 479       return false;
 480
 481    // Stash the / terminated directory prefix
 482    string Base = Section->FindS("Directory");
 483    if (Base.empty() == false && Base[Base.length()-1] != '/')
 484       Base += '/';
 485
 486    // Read the first file triplet
 487    const char *C = Files.c_str();
 488    string sSize;
 489    string MD5Hash;
 490
 491    // Parse each of the elements
 492    if (ParseQuoteWord(C,MD5Hash) == false ||
 493        ParseQuoteWord(C,sSize) == false ||
 494        ParseQuoteWord(C,File) == false)
 495       return _error->Error("Error parsing file record");
 496
 497    // Parse the size and append the directory
 498    Size = atoi(sSize.c_str());
 499    File = Base + File;
 500    return true;
 501 }
 502                                                                         /*}}}*/
 503 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 504 // ---------------------------------------------------------------------
 505 /* */
 506 bool SourceCopy::RewriteEntry(FILE *Target,string File)
 507 {
 508    string Dir(File,0,File.rfind('/'));
 509    TFRewriteData Changes[] = {{"Directory",Dir.c_str()},
 510                               {}};
 511
 512    if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
 513       return false;
 514    fputc('\n',Target);
 515    return true;
 516 }
 517                                                                         /*}}}*/
 518 // SigVerify::Verify - Verify a files md5sum against its metaindex      /*{{{*/
 519 // ---------------------------------------------------------------------
 520 /* */
 521 bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex)
 522 {
 523    const indexRecords::checkSum *Record = MetaIndex->Lookup(file);
 524
 525    if (!Record)
 526    {
 527       _error->Warning("Can't find authentication record for: %s",file.c_str());
 528       return false;
 529    }
 530
 531    if (!Record->Hash.VerifyFile(prefix+file))
 532    {
 533       _error->Warning("Hash mismatch for: %s",file.c_str());
 534       return false;
 535    }
 536
 537    if(_config->FindB("Debug::aptcdrom",false))
 538    {
 539       cout << "File: " << prefix+file << endl;
 540       cout << "Expected Hash " << Record->Hash.toStr() << endl;
 541    }
 542
 543    return true;
 544 }
 545
 546 bool SigVerify::CopyMetaIndex(string CDROM, string CDName,
 547                               string prefix, string file)
 548 {
 549       char S[400];
 550       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",CDName.c_str(),
 551                (prefix).c_str() + CDROM.length(),file.c_str());
 552       string TargetF = _config->FindDir("Dir::State::lists");
 553       TargetF += URItoFileName(S);
 554
 555       FileFd Target;
 556       FileFd Rel;
 557       Target.Open(TargetF,FileFd::WriteEmpty);
 558       Rel.Open(prefix + file,FileFd::ReadOnly);
 559       if (_error->PendingError() == true)
 560          return false;
 561       if (CopyFile(Rel,Target) == false)
 562          return false;
 563
 564       return true;
 565 }
 566
 567 bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList,
 568                               vector<string> PkgList,vector<string> SrcList)
 569 {
 570    if (SigList.size() == 0)
 571       return true;
 572
 573    bool Debug = _config->FindB("Debug::aptcdrom",false);
 574
 575    // Read all Release files
 576    for (vector<string>::iterator I = SigList.begin(); I != SigList.end(); I++)
 577    {
 578       if(Debug)
 579          cout << "Signature verify for: " << *I << endl;
 580
 581       indexRecords *MetaIndex = new indexRecords;
 582       string prefix = *I;
 583
 584       // a Release.gpg without a Release should never happen
 585       if(!FileExists(*I+"Release"))
 586          continue;
 587
 588
 589       // verify the gpg signature of "Release"
 590       // gpg --verify "*I+Release.gpg", "*I+Release"
 591       const char *Args[400];
 592       unsigned int i = 0;
 593
 594       string gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv");
 595       string pubringpath = _config->Find("Apt::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg");
 596       string releasegpg = *I+"Release.gpg";
 597       string release = *I+"Release";
 598
 599       Args[i++] = gpgvpath.c_str();
 600       Args[i++] = "--keyring";
 601       Args[i++] = pubringpath.c_str();
 602       Configuration::Item const *Opts;
 603       Opts = _config->Tree("Acquire::gpgv::Options");
 604       if (Opts != 0)
 605       {
 606          Opts = Opts->Child;
 607          for (; Opts != 0; Opts = Opts->Next)
 608          {
 609             if (Opts->Value.empty() == true)
 610                continue;
 611             Args[i++] = Opts->Value.c_str();
 612             if(i >= 390) {
 613                _error->Error("Argument list from Acquire::gpgv::Options too long. Exiting.");
 614                return false;
 615             }
 616          }
 617       }
 618
 619       Args[i++] = releasegpg.c_str();
 620       Args[i++] = release.c_str();
 621       Args[i++] = NULL;
 622
 623       pid_t pid = ExecFork();
 624       if(pid < 0) {
 625          _error->Error("Fork failed");
 626          return false;
 627       }
 628       if(pid == 0) {
 629          execvp(gpgvpath.c_str(), (char**)Args);
 630       }
 631       if(!ExecWait(pid, "gpgv")) {
 632          _error->Warning("Signature verification failed for: %s",
 633                          string(*I+"Release.gpg").c_str());
 634          // something went wrong, don't copy the Release.gpg
 635          // FIXME: delete any existing gpg file?
 636          continue;
 637       }
 638
 639       // Open the Release file and add it to the MetaIndex
 640       if(!MetaIndex->Load(*I+"Release"))
 641       {
 642          _error->Error("%s",MetaIndex->ErrorText.c_str());
 643          return false;
 644       }
 645
 646       // go over the Indexfiles and see if they verify
 647       // if so, remove them from our copy of the lists
 648       vector<string> keys = MetaIndex->MetaKeys();
 649       for (vector<string>::iterator I = keys.begin(); I != keys.end(); I++)
 650       {
 651          if(!Verify(prefix,*I, MetaIndex)) {
 652             // something went wrong, don't copy the Release.gpg
 653             // FIXME: delete any existing gpg file?
 654             continue;
 655          }
 656       }
 657
 658       // we need a fresh one for the Release.gpg
 659       delete MetaIndex;
 660
 661       // everything was fine, copy the Release and Release.gpg file
 662       CopyMetaIndex(CDROM, Name, prefix, "Release");
 663       CopyMetaIndex(CDROM, Name, prefix, "Release.gpg");
 664    }
 665
 666    return true;
 667 }
 668
 669
 670 bool TranslationsCopy::CopyTranslations(string CDROM,string Name,vector<string> &List,
 671                              pkgCdromStatus *log)
 672 {
 673    OpProgress *Progress = NULL;
 674    if (List.size() == 0)
 675       return true;
 676
 677    if(log)
 678       Progress = log->GetOpProgress();
 679
 680    bool Debug = _config->FindB("Debug::aptcdrom",false);
 681
 682    // Prepare the progress indicator
 683    unsigned long TotalSize = 0;
 684    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
 685    {
 686       struct stat Buf;
 687       if (stat(string(*I).c_str(),&Buf) != 0 &&
 688           stat(string(*I + ".gz").c_str(),&Buf) != 0)
 689          return _error->Errno("stat","Stat failed for %s",
 690                               string(*I).c_str());
 691       TotalSize += Buf.st_size;
 692    }
 693
 694    unsigned long CurrentSize = 0;
 695    unsigned int NotFound = 0;
 696    unsigned int WrongSize = 0;
 697    unsigned int Packages = 0;
 698    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
 699    {
 700       string OrigPath = string(*I,CDROM.length());
 701       unsigned long FileSize = 0;
 702
 703       // Open the package file
 704       FileFd Pkg;
 705       if (FileExists(*I) == true)
 706       {
 707          Pkg.Open(*I,FileFd::ReadOnly);
 708          FileSize = Pkg.Size();
 709       }
 710       else
 711       {
 712          FileFd From(*I + ".gz",FileFd::ReadOnly);
 713          if (_error->PendingError() == true)
 714             return false;
 715          FileSize = From.Size();
 716
 717          // Get a temp file
 718          FILE *tmp = tmpfile();
 719          if (tmp == 0)
 720             return _error->Errno("tmpfile","Unable to create a tmp file");
 721          Pkg.Fd(dup(fileno(tmp)));
 722          fclose(tmp);
 723
 724          // Fork gzip
 725          pid_t Process = fork();
 726          if (Process < 0)
 727             return _error->Errno("fork","Couldn't fork gzip");
 728
 729          // The child
 730          if (Process == 0)
 731          {
 732             dup2(From.Fd(),STDIN_FILENO);
 733             dup2(Pkg.Fd(),STDOUT_FILENO);
 734             SetCloseExec(STDIN_FILENO,false);
 735             SetCloseExec(STDOUT_FILENO,false);
 736
 737             const char *Args[3];
 738             string Tmp =  _config->Find("Dir::bin::gzip","gzip");
 739             Args[0] = Tmp.c_str();
 740             Args[1] = "-d";
 741             Args[2] = 0;
 742             execvp(Args[0],(char **)Args);
 743             exit(100);
 744          }
 745
 746          // Wait for gzip to finish
 747          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 748             return _error->Error("gzip failed, perhaps the disk is full.");
 749
 750          Pkg.Seek(0);
 751       }
 752       pkgTagFile Parser(&Pkg);
 753       if (_error->PendingError() == true)
 754          return false;
 755
 756       // Open the output file
 757       char S[400];
 758       snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(),
 759                (*I).c_str() + CDROM.length());
 760       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 761       TargetF += URItoFileName(S);
 762       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 763          TargetF = "/dev/null";
 764       FileFd Target(TargetF,FileFd::WriteEmpty);
 765       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 766       if (_error->PendingError() == true)
 767          return false;
 768       if (TargetFl == 0)
 769          return _error->Errno("fdopen","Failed to reopen fd");
 770
 771       // Setup the progress meter
 772       if(Progress)
 773          Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
 774                                    string("Reading Translation Indexes"));
 775
 776       // Parse
 777       if(Progress)
 778          Progress->SubProgress(Pkg.Size());
 779       pkgTagSection Section;
 780       this->Section = &Section;
 781       string Prefix;
 782       unsigned long Hits = 0;
 783       unsigned long Chop = 0;
 784       while (Parser.Step(Section) == true)
 785       {
 786          if(Progress)
 787             Progress->Progress(Parser.Offset());
 788
 789          const char *Start;
 790          const char *Stop;
 791          Section.GetSection(Start,Stop);
 792          fwrite(Start,Stop-Start, 1, TargetFl);
 793          fputc('\n',TargetFl);
 794
 795          Packages++;
 796          Hits++;
 797       }
 798       fclose(TargetFl);
 799
 800       if (Debug == true)
 801          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 802
 803       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 804       {
 805          // Move out of the partial directory
 806          Target.Close();
 807          string FinalF = _config->FindDir("Dir::State::lists");
 808          FinalF += URItoFileName(S);
 809          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 810             return _error->Errno("rename","Failed to rename");
 811       }
 812
 813
 814       CurrentSize += FileSize;
 815    }
 816    if(Progress)
 817       Progress->Done();
 818
 819    // Some stats
 820    if(log) {
 821       stringstream msg;
 822       if(NotFound == 0 && WrongSize == 0)
 823          ioprintf(msg, _("Wrote %i records.\n"), Packages);
 824       else if (NotFound != 0 && WrongSize == 0)
 825          ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 826                   Packages, NotFound);
 827       else if (NotFound == 0 && WrongSize != 0)
 828          ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 829                   Packages, WrongSize);
 830       if (NotFound != 0 && WrongSize != 0)
 831          ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
 832    }
 833
 834    if (Packages == 0)
 835       _error->Warning("No valid records were found.");
 836
 837    if (NotFound + WrongSize > 10)
 838       _error->Warning("Alot of entries were discarded, something may be wrong.\n");
 839
 840
 841    return true;
 842 }