apt-pkg/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.10 2002/03/26 07:38:58 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/indexrecords.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/cdrom.h>
  24 #include <apti18n.h>
  25
  26 #include <iostream>
  27 #include <sstream>
  28 #include <unistd.h>
  29 #include <sys/stat.h>
  30 #include <stdio.h>
  31                                                                         /*}}}*/
  32
  33 using namespace std;
  34
  35
  36
  37 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  38 // ---------------------------------------------------------------------
  39 /* */
  40 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
  41                              pkgCdromStatus *log)
  42 {
  43    OpProgress *Progress = NULL;
  44    if (List.size() == 0)
  45       return true;
  46
  47    if(log)
  48       Progress = log->GetOpProgress();
  49
  50    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  51    bool Debug = _config->FindB("Debug::aptcdrom",false);
  52
  53    // Prepare the progress indicator
  54    unsigned long TotalSize = 0;
  55    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  56    {
  57       struct stat Buf;
  58       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  59           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  60          return _error->Errno("stat","Stat failed for %s",
  61                               string(*I + GetFileName()).c_str());
  62       TotalSize += Buf.st_size;
  63    }
  64
  65    unsigned long CurrentSize = 0;
  66    unsigned int NotFound = 0;
  67    unsigned int WrongSize = 0;
  68    unsigned int Packages = 0;
  69    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  70    {
  71       string OrigPath = string(*I,CDROM.length());
  72       unsigned long FileSize = 0;
  73
  74       // Open the package file
  75       FileFd Pkg;
  76       if (FileExists(*I + GetFileName()) == true)
  77       {
  78          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  79          FileSize = Pkg.Size();
  80       }
  81       else
  82       {
  83          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  84          if (_error->PendingError() == true)
  85             return false;
  86          FileSize = From.Size();
  87
  88          // Get a temp file
  89          FILE *tmp = tmpfile();
  90          if (tmp == 0)
  91             return _error->Errno("tmpfile","Unable to create a tmp file");
  92          Pkg.Fd(dup(fileno(tmp)));
  93          fclose(tmp);
  94
  95          // Fork gzip
  96          pid_t Process = fork();
  97          if (Process < 0)
  98             return _error->Errno("fork","Couldn't fork gzip");
  99
 100          // The child
 101          if (Process == 0)
 102          {
 103             dup2(From.Fd(),STDIN_FILENO);
 104             dup2(Pkg.Fd(),STDOUT_FILENO);
 105             SetCloseExec(STDIN_FILENO,false);
 106             SetCloseExec(STDOUT_FILENO,false);
 107
 108             const char *Args[3];
 109             string Tmp =  _config->Find("Dir::bin::gzip","gzip");
 110             Args[0] = Tmp.c_str();
 111             Args[1] = "-d";
 112             Args[2] = 0;
 113             execvp(Args[0],(char **)Args);
 114             exit(100);
 115          }
 116
 117          // Wait for gzip to finish
 118          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 119             return _error->Error("gzip failed, perhaps the disk is full.");
 120
 121          Pkg.Seek(0);
 122       }
 123       pkgTagFile Parser(&Pkg);
 124       if (_error->PendingError() == true)
 125          return false;
 126
 127       // Open the output file
 128       char S[400];
 129       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",Name.c_str(),
 130                (*I).c_str() + CDROM.length(),GetFileName());
 131       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 132       TargetF += URItoFileName(S);
 133       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 134          TargetF = "/dev/null";
 135       FileFd Target(TargetF,FileFd::WriteEmpty);
 136       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 137       if (_error->PendingError() == true)
 138          return false;
 139       if (TargetFl == 0)
 140          return _error->Errno("fdopen","Failed to reopen fd");
 141
 142       // Setup the progress meter
 143       if(Progress)
 144          Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
 145                                    string("Reading ") + Type() + " Indexes");
 146
 147       // Parse
 148       if(Progress)
 149          Progress->SubProgress(Pkg.Size());
 150       pkgTagSection Section;
 151       this->Section = &Section;
 152       string Prefix;
 153       unsigned long Hits = 0;
 154       unsigned long Chop = 0;
 155       while (Parser.Step(Section) == true)
 156       {
 157          if(Progress)
 158             Progress->Progress(Parser.Offset());
 159          string File;
 160          unsigned long Size;
 161          if (GetFile(File,Size) == false)
 162          {
 163             fclose(TargetFl);
 164             return false;
 165          }
 166
 167          if (Chop != 0)
 168             File = OrigPath + ChopDirs(File,Chop);
 169
 170          // See if the file exists
 171          bool Mangled = false;
 172          if (NoStat == false || Hits < 10)
 173          {
 174             // Attempt to fix broken structure
 175             if (Hits == 0)
 176             {
 177                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 178                    ReconstructChop(Chop,*I,File) == false)
 179                {
 180                   if (Debug == true)
 181                      clog << "Missed: " << File << endl;
 182                   NotFound++;
 183                   continue;
 184                }
 185                if (Chop != 0)
 186                   File = OrigPath + ChopDirs(File,Chop);
 187             }
 188
 189             // Get the size
 190             struct stat Buf;
 191             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 192                 Buf.st_size == 0)
 193             {
 194                // Attempt to fix busted symlink support for one instance
 195                string OrigFile = File;
 196                string::size_type Start = File.find("binary-");
 197                string::size_type End = File.find("/",Start+3);
 198                if (Start != string::npos && End != string::npos)
 199                {
 200                   File.replace(Start,End-Start,"binary-all");
 201                   Mangled = true;
 202                }
 203
 204                if (Mangled == false ||
 205                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 206                {
 207                   if (Debug == true)
 208                      clog << "Missed(2): " << OrigFile << endl;
 209                   NotFound++;
 210                   continue;
 211                }
 212             }
 213
 214             // Size match
 215             if ((unsigned)Buf.st_size != Size)
 216             {
 217                if (Debug == true)
 218                   clog << "Wrong Size: " << File << endl;
 219                WrongSize++;
 220                continue;
 221             }
 222          }
 223
 224          Packages++;
 225          Hits++;
 226
 227          if (RewriteEntry(TargetFl,File) == false)
 228          {
 229             fclose(TargetFl);
 230             return false;
 231          }
 232       }
 233       fclose(TargetFl);
 234
 235       if (Debug == true)
 236          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 237
 238       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 239       {
 240          // Move out of the partial directory
 241          Target.Close();
 242          string FinalF = _config->FindDir("Dir::State::lists");
 243          FinalF += URItoFileName(S);
 244          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 245             return _error->Errno("rename","Failed to rename");
 246       }
 247
 248       /* Mangle the source to be in the proper notation with
 249          prefix dist [component] */
 250       *I = string(*I,Prefix.length());
 251       ConvertToSourceList(CDROM,*I);
 252       *I = Prefix + ' ' + *I;
 253
 254       CurrentSize += FileSize;
 255    }
 256    if(Progress)
 257       Progress->Done();
 258
 259    // Some stats
 260    if(log) {
 261       stringstream msg;
 262       if(NotFound == 0 && WrongSize == 0)
 263          ioprintf(msg, _("Wrote %i records.\n"), Packages);
 264       else if (NotFound != 0 && WrongSize == 0)
 265          ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 266                   Packages, NotFound);
 267       else if (NotFound == 0 && WrongSize != 0)
 268          ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 269                   Packages, WrongSize);
 270       if (NotFound != 0 && WrongSize != 0)
 271          ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
 272    }
 273
 274    if (Packages == 0)
 275       _error->Warning("No valid records were found.");
 276
 277    if (NotFound + WrongSize > 10)
 278       _error->Warning("Alot of entries were discarded, something may be wrong.\n");
 279
 280
 281    return true;
 282 }
 283                                                                         /*}}}*/
 284 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 285 // ---------------------------------------------------------------------
 286 /* */
 287 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 288 {
 289    string::size_type I = 0;
 290    do
 291    {
 292       I = Path.find('/',I+1);
 293       Depth--;
 294    }
 295    while (I != string::npos && Depth != 0);
 296
 297    if (I == string::npos)
 298       return string();
 299
 300    return string(Path,I+1);
 301 }
 302                                                                         /*}}}*/
 303 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 304 // ---------------------------------------------------------------------
 305 /* This prepends dir components from the path to the package files to
 306    the path to the deb until it is found */
 307 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 308                                   string File)
 309 {
 310    bool Debug = _config->FindB("Debug::aptcdrom",false);
 311    unsigned int Depth = 1;
 312    string MyPrefix = Prefix;
 313    while (1)
 314    {
 315       struct stat Buf;
 316       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 317       {
 318          if (Debug == true)
 319             cout << "Failed, " << CD + MyPrefix + File << endl;
 320          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 321             continue;
 322
 323          return false;
 324       }
 325       else
 326       {
 327          Prefix = MyPrefix;
 328          return true;
 329       }
 330    }
 331    return false;
 332 }
 333                                                                         /*}}}*/
 334 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 335 // ---------------------------------------------------------------------
 336 /* This removes path components from the filename and prepends the location
 337    of the package files until a file is found */
 338 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 339 {
 340    // Attempt to reconstruct the filename
 341    unsigned long Depth = 0;
 342    while (1)
 343    {
 344       struct stat Buf;
 345       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 346       {
 347          File = ChopDirs(File,1);
 348          Depth++;
 349          if (File.empty() == false)
 350             continue;
 351          return false;
 352       }
 353       else
 354       {
 355          Chop = Depth;
 356          return true;
 357       }
 358    }
 359    return false;
 360 }
 361                                                                         /*}}}*/
 362 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 363 // ---------------------------------------------------------------------
 364 /* We look for things in dists/ notation and convert them to
 365    <dist> <component> form otherwise it is left alone. This also strips
 366    the CD path.
 367
 368    This implements a regex sort of like:
 369     (.*)/dists/([^/]*)/(.*)/binary-*
 370      ^          ^      ^- Component
 371      |          |-------- Distribution
 372      |------------------- Path
 373
 374    It was deciced to use only a single word for dist (rather than say
 375    unstable/non-us) to increase the chance that each CD gets a single
 376    line in sources.list.
 377  */
 378 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 379 {
 380    char S[300];
 381    snprintf(S,sizeof(S),"binary-%s",_config->Find("Apt::Architecture").c_str());
 382
 383    // Strip the cdrom base path
 384    Path = string(Path,CD.length());
 385    if (Path.empty() == true)
 386       Path = "/";
 387
 388    // Too short to be a dists/ type
 389    if (Path.length() < strlen("dists/"))
 390       return;
 391
 392    // Not a dists type.
 393    if (stringcmp(Path.c_str(),Path.c_str()+strlen("dists/"),"dists/") != 0)
 394       return;
 395
 396    // Isolate the dist
 397    string::size_type Slash = strlen("dists/");
 398    string::size_type Slash2 = Path.find('/',Slash + 1);
 399    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 400       return;
 401    string Dist = string(Path,Slash,Slash2 - Slash);
 402
 403    // Isolate the component
 404    Slash = Slash2;
 405    for (unsigned I = 0; I != 10; I++)
 406    {
 407       Slash = Path.find('/',Slash+1);
 408       if (Slash == string::npos || Slash + 2 >= Path.length())
 409          return;
 410       string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 411
 412       // Verify the trailing binary- bit
 413       string::size_type BinSlash = Path.find('/',Slash + 1);
 414       if (Slash == string::npos)
 415          return;
 416       string Binary = string(Path,Slash+1,BinSlash - Slash-1);
 417
 418       if (Binary != S && Binary != "source")
 419          continue;
 420
 421       Path = Dist + ' ' + Comp;
 422       return;
 423    }
 424 }
 425                                                                         /*}}}*/
 426 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 427 // ---------------------------------------------------------------------
 428 /* */
 429 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 430 {
 431    string::size_type I = 0;
 432    do
 433    {
 434       I = Path.find('/',I+1);
 435       Depth--;
 436    }
 437    while (I != string::npos && Depth != 0);
 438
 439    if (I == string::npos)
 440       return false;
 441
 442    To = string(Path,0,I+1);
 443    return true;
 444 }
 445                                                                         /*}}}*/
 446 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* */
 449 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 450 {
 451    File = Section->FindS("Filename");
 452    Size = Section->FindI("Size");
 453    if (File.empty() || Size == 0)
 454       return _error->Error("Cannot find filename or size tag");
 455    return true;
 456 }
 457                                                                         /*}}}*/
 458 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* */
 461 bool PackageCopy::RewriteEntry(FILE *Target,string File)
 462 {
 463    TFRewriteData Changes[] = {{"Filename",File.c_str()},
 464                               {}};
 465
 466    if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
 467       return false;
 468    fputc('\n',Target);
 469    return true;
 470 }
 471                                                                         /*}}}*/
 472 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* */
 475 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 476 {
 477    string Files = Section->FindS("Files");
 478    if (Files.empty() == true)
 479       return false;
 480
 481    // Stash the / terminated directory prefix
 482    string Base = Section->FindS("Directory");
 483    if (Base.empty() == false && Base[Base.length()-1] != '/')
 484       Base += '/';
 485
 486    // Read the first file triplet
 487    const char *C = Files.c_str();
 488    string sSize;
 489    string MD5Hash;
 490
 491    // Parse each of the elements
 492    if (ParseQuoteWord(C,MD5Hash) == false ||
 493        ParseQuoteWord(C,sSize) == false ||
 494        ParseQuoteWord(C,File) == false)
 495       return _error->Error("Error parsing file record");
 496
 497    // Parse the size and append the directory
 498    Size = atoi(sSize.c_str());
 499    File = Base + File;
 500    return true;
 501 }
 502                                                                         /*}}}*/
 503 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 504 // ---------------------------------------------------------------------
 505 /* */
 506 bool SourceCopy::RewriteEntry(FILE *Target,string File)
 507 {
 508    string Dir(File,0,File.rfind('/'));
 509    TFRewriteData Changes[] = {{"Directory",Dir.c_str()},
 510                               {}};
 511
 512    if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
 513       return false;
 514    fputc('\n',Target);
 515    return true;
 516 }
 517                                                                         /*}}}*/
 518 // SigVerify::Verify - Verify a files md5sum against its metaindex      /*{{{*/
 519 // ---------------------------------------------------------------------
 520 /* */
 521 bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex)
 522 {
 523    const indexRecords::checkSum *Record = MetaIndex->Lookup(file);
 524
 525    if (!Record)
 526    {
 527       _error->Warning("Can't find authentication record for: %s",file.c_str());
 528       return false;
 529    }
 530
 531    MD5Summation sum;
 532    FileFd Fd(prefix+file, FileFd::ReadOnly);
 533    sum.AddFD(Fd.Fd(), Fd.Size());
 534    Fd.Close();
 535    string MD5 = (string)sum.Result();
 536
 537    if (Record->MD5Hash != MD5)
 538    {
 539       _error->Warning("MD5 mismatch for: %s",file.c_str());
 540       return false;
 541    }
 542
 543    if(_config->FindB("Debug::aptcdrom",false))
 544    {
 545       cout << "File: " << prefix+file << endl;
 546       cout << "Expected MD5sum: " << Record->MD5Hash << endl;
 547       cout << "got: " << MD5 << endl << endl;
 548    }
 549
 550    return true;
 551 }
 552
 553 bool SigVerify::CopyMetaIndex(string CDROM, string CDName,
 554                               string prefix, string file)
 555 {
 556       char S[400];
 557       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",CDName.c_str(),
 558                (prefix).c_str() + CDROM.length(),file.c_str());
 559       string TargetF = _config->FindDir("Dir::State::lists");
 560       TargetF += URItoFileName(S);
 561
 562       FileFd Target;
 563       FileFd Rel;
 564       Target.Open(TargetF,FileFd::WriteEmpty);
 565       Rel.Open(prefix + file,FileFd::ReadOnly);
 566       if (_error->PendingError() == true)
 567          return false;
 568       if (CopyFile(Rel,Target) == false)
 569          return false;
 570
 571       return true;
 572 }
 573
 574 bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList,
 575                               vector<string> PkgList,vector<string> SrcList)
 576 {
 577    if (SigList.size() == 0)
 578       return true;
 579
 580    bool Debug = _config->FindB("Debug::aptcdrom",false);
 581
 582    // Read all Release files
 583    for (vector<string>::iterator I = SigList.begin(); I != SigList.end(); I++)
 584    {
 585       if(Debug)
 586          cout << "Signature verify for: " << *I << endl;
 587
 588       indexRecords *MetaIndex = new indexRecords;
 589       string prefix = *I;
 590
 591       // a Release.gpg without a Release should never happen
 592       if(!FileExists(*I+"Release"))
 593          continue;
 594
 595
 596       // verify the gpg signature of "Release"
 597       // gpg --verify "*I+Release.gpg", "*I+Release"
 598       const char *Args[400];
 599       unsigned int i = 0;
 600
 601       string gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv");
 602       string pubringpath = _config->Find("Apt::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg");
 603       string releasegpg = *I+"Release.gpg";
 604       string release = *I+"Release";
 605
 606       Args[i++] = gpgvpath.c_str();
 607       Args[i++] = "--keyring";
 608       Args[i++] = pubringpath.c_str();
 609       Configuration::Item const *Opts;
 610       Opts = _config->Tree("Acquire::gpgv::Options");
 611       if (Opts != 0)
 612       {
 613          Opts = Opts->Child;
 614          for (; Opts != 0; Opts = Opts->Next)
 615          {
 616             if (Opts->Value.empty() == true)
 617                continue;
 618             Args[i++] = Opts->Value.c_str();
 619             if(i >= 390) {
 620                _error->Error("Argument list from Acquire::gpgv::Options too long. Exiting.");
 621                return false;
 622             }
 623          }
 624       }
 625
 626       Args[i++] = releasegpg.c_str();
 627       Args[i++] = release.c_str();
 628       Args[i++] = NULL;
 629
 630       pid_t pid = ExecFork();
 631       if(pid < 0) {
 632          _error->Error("Fork failed");
 633          return false;
 634       }
 635       if(pid == 0) {
 636          execvp(gpgvpath.c_str(), (char**)Args);
 637       }
 638       if(!ExecWait(pid, "gpgv")) {
 639          _error->Warning("Signature verification failed for: %s",
 640                          string(*I+"Release.gpg").c_str());
 641          // something went wrong, don't copy the Release.gpg
 642          // FIXME: delete any existing gpg file?
 643          continue;
 644       }
 645
 646       // Open the Release file and add it to the MetaIndex
 647       if(!MetaIndex->Load(*I+"Release"))
 648       {
 649          _error->Error(MetaIndex->ErrorText.c_str());
 650          return false;
 651       }
 652
 653       // go over the Indexfiles and see if they verify
 654       // if so, remove them from our copy of the lists
 655       vector<string> keys = MetaIndex->MetaKeys();
 656       for (vector<string>::iterator I = keys.begin(); I != keys.end(); I++)
 657       {
 658          if(!Verify(prefix,*I, MetaIndex)) {
 659             // something went wrong, don't copy the Release.gpg
 660             // FIXME: delete any existing gpg file?
 661             continue;
 662          }
 663       }
 664
 665       // we need a fresh one for the Release.gpg
 666       delete MetaIndex;
 667
 668       // everything was fine, copy the Release and Release.gpg file
 669       CopyMetaIndex(CDROM, Name, prefix, "Release");
 670       CopyMetaIndex(CDROM, Name, prefix, "Release.gpg");
 671    }
 672
 673    return true;
 674 }
 675
 676
 677 bool TranslationsCopy::CopyTranslations(string CDROM,string Name,vector<string> &List,
 678                              pkgCdromStatus *log)
 679 {
 680    OpProgress *Progress = NULL;
 681    if (List.size() == 0)
 682       return true;
 683
 684    if(log)
 685       Progress = log->GetOpProgress();
 686
 687    bool Debug = _config->FindB("Debug::aptcdrom",false);
 688
 689    // Prepare the progress indicator
 690    unsigned long TotalSize = 0;
 691    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
 692    {
 693       struct stat Buf;
 694       if (stat(string(*I).c_str(),&Buf) != 0 &&
 695           stat(string(*I + ".gz").c_str(),&Buf) != 0)
 696          return _error->Errno("stat","Stat failed for %s",
 697                               string(*I).c_str());
 698       TotalSize += Buf.st_size;
 699    }
 700
 701    unsigned long CurrentSize = 0;
 702    unsigned int NotFound = 0;
 703    unsigned int WrongSize = 0;
 704    unsigned int Packages = 0;
 705    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
 706    {
 707       string OrigPath = string(*I,CDROM.length());
 708       unsigned long FileSize = 0;
 709
 710       // Open the package file
 711       FileFd Pkg;
 712       if (FileExists(*I) == true)
 713       {
 714          Pkg.Open(*I,FileFd::ReadOnly);
 715          FileSize = Pkg.Size();
 716       }
 717       else
 718       {
 719          FileFd From(*I + ".gz",FileFd::ReadOnly);
 720          if (_error->PendingError() == true)
 721             return false;
 722          FileSize = From.Size();
 723
 724          // Get a temp file
 725          FILE *tmp = tmpfile();
 726          if (tmp == 0)
 727             return _error->Errno("tmpfile","Unable to create a tmp file");
 728          Pkg.Fd(dup(fileno(tmp)));
 729          fclose(tmp);
 730
 731          // Fork gzip
 732          pid_t Process = fork();
 733          if (Process < 0)
 734             return _error->Errno("fork","Couldn't fork gzip");
 735
 736          // The child
 737          if (Process == 0)
 738          {
 739             dup2(From.Fd(),STDIN_FILENO);
 740             dup2(Pkg.Fd(),STDOUT_FILENO);
 741             SetCloseExec(STDIN_FILENO,false);
 742             SetCloseExec(STDOUT_FILENO,false);
 743
 744             const char *Args[3];
 745             string Tmp =  _config->Find("Dir::bin::gzip","gzip");
 746             Args[0] = Tmp.c_str();
 747             Args[1] = "-d";
 748             Args[2] = 0;
 749             execvp(Args[0],(char **)Args);
 750             exit(100);
 751          }
 752
 753          // Wait for gzip to finish
 754          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 755             return _error->Error("gzip failed, perhaps the disk is full.");
 756
 757          Pkg.Seek(0);
 758       }
 759       pkgTagFile Parser(&Pkg);
 760       if (_error->PendingError() == true)
 761          return false;
 762
 763       // Open the output file
 764       char S[400];
 765       snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(),
 766                (*I).c_str() + CDROM.length());
 767       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 768       TargetF += URItoFileName(S);
 769       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 770          TargetF = "/dev/null";
 771       FileFd Target(TargetF,FileFd::WriteEmpty);
 772       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 773       if (_error->PendingError() == true)
 774          return false;
 775       if (TargetFl == 0)
 776          return _error->Errno("fdopen","Failed to reopen fd");
 777
 778       // Setup the progress meter
 779       if(Progress)
 780          Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
 781                                    string("Reading Translation Indexes"));
 782
 783       // Parse
 784       if(Progress)
 785          Progress->SubProgress(Pkg.Size());
 786       pkgTagSection Section;
 787       this->Section = &Section;
 788       string Prefix;
 789       unsigned long Hits = 0;
 790       unsigned long Chop = 0;
 791       while (Parser.Step(Section) == true)
 792       {
 793          if(Progress)
 794             Progress->Progress(Parser.Offset());
 795
 796          const char *Start;
 797          const char *Stop;
 798          Section.GetSection(Start,Stop);
 799          fwrite(Start,Stop-Start, 1, TargetFl);
 800          fputc('\n',TargetFl);
 801
 802          Packages++;
 803          Hits++;
 804       }
 805       fclose(TargetFl);
 806
 807       if (Debug == true)
 808          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 809
 810       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 811       {
 812          // Move out of the partial directory
 813          Target.Close();
 814          string FinalF = _config->FindDir("Dir::State::lists");
 815          FinalF += URItoFileName(S);
 816          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 817             return _error->Errno("rename","Failed to rename");
 818       }
 819
 820
 821       CurrentSize += FileSize;
 822    }
 823    if(Progress)
 824       Progress->Done();
 825
 826    // Some stats
 827    if(log) {
 828       stringstream msg;
 829       if(NotFound == 0 && WrongSize == 0)
 830          ioprintf(msg, _("Wrote %i records.\n"), Packages);
 831       else if (NotFound != 0 && WrongSize == 0)
 832          ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 833                   Packages, NotFound);
 834       else if (NotFound == 0 && WrongSize != 0)
 835          ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 836                   Packages, WrongSize);
 837       if (NotFound != 0 && WrongSize != 0)
 838          ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
 839    }
 840
 841    if (Packages == 0)
 842       _error->Warning("No valid records were found.");
 843
 844    if (NotFound + WrongSize > 10)
 845       _error->Warning("Alot of entries were discarded, something may be wrong.\n");
 846
 847
 848    return true;
 849 }