cmdline/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.2 1999/09/03 05:46:48 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21
  22 #include <iostream.h>
  23 #include <unistd.h>
  24 #include <sys/stat.h>
  25 #include <stdio.h>
  26 #include <wait.h>
  27                                                                         /*}}}*/
  28
  29 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  30 // ---------------------------------------------------------------------
  31 /* */
  32 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List)
  33 {
  34    if (List.size() == 0)
  35       return true;
  36
  37    OpTextProgress Progress;
  38
  39    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  40    bool Debug = _config->FindB("Debug::aptcdrom",false);
  41
  42    // Prepare the progress indicator
  43    unsigned long TotalSize = 0;
  44    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  45    {
  46       struct stat Buf;
  47       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  48           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  49          return _error->Errno("stat","Stat failed for %s",
  50                               string(*I + GetFileName()).c_str());
  51       TotalSize += Buf.st_size;
  52    }
  53
  54    unsigned long CurrentSize = 0;
  55    unsigned int NotFound = 0;
  56    unsigned int WrongSize = 0;
  57    unsigned int Packages = 0;
  58    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  59    {
  60       string OrigPath = string(*I,CDROM.length());
  61       unsigned long FileSize = 0;
  62
  63       // Open the package file
  64       FileFd Pkg;
  65       if (FileExists(*I + GetFileName()) == true)
  66       {
  67          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  68          FileSize = Pkg.Size();
  69       }
  70       else
  71       {
  72          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  73          if (_error->PendingError() == true)
  74             return false;
  75          FileSize = From.Size();
  76
  77          // Get a temp file
  78          FILE *tmp = tmpfile();
  79          if (tmp == 0)
  80             return _error->Errno("tmpfile","Unable to create a tmp file");
  81          Pkg.Fd(dup(fileno(tmp)));
  82          fclose(tmp);
  83
  84          // Fork gzip
  85          int Process = fork();
  86          if (Process < 0)
  87             return _error->Errno("fork","Couldn't fork gzip");
  88
  89          // The child
  90          if (Process == 0)
  91          {
  92             dup2(From.Fd(),STDIN_FILENO);
  93             dup2(Pkg.Fd(),STDOUT_FILENO);
  94             SetCloseExec(STDIN_FILENO,false);
  95             SetCloseExec(STDOUT_FILENO,false);
  96
  97             const char *Args[3];
  98             Args[0] = _config->Find("Dir::bin::gzip","gzip").c_str();
  99             Args[1] = "-d";
 100             Args[2] = 0;
 101             execvp(Args[0],(char **)Args);
 102             exit(100);
 103          }
 104
 105          // Wait for gzip to finish
 106          int Status;
 107          if (waitpid(Process,&Status,0) != Process)
 108             return _error->Errno("wait","Waiting for gzip failed");
 109          if (WIFEXITED(Status) == 0 || WEXITSTATUS(Status) != 0)
 110             return _error->Error("gzip failed, perhaps the disk is full.");
 111          Pkg.Seek(0);
 112       }
 113       pkgTagFile Parser(Pkg);
 114       if (_error->PendingError() == true)
 115          return false;
 116
 117       // Open the output file
 118       char S[400];
 119       sprintf(S,"cdrom:%s/%s%s",Name.c_str(),(*I).c_str() + CDROM.length(),
 120               GetFileName());
 121       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 122       TargetF += URItoFileName(S);
 123       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 124          TargetF = "/dev/null";
 125       FileFd Target(TargetF,FileFd::WriteEmpty);
 126       if (_error->PendingError() == true)
 127          return false;
 128
 129       // Setup the progress meter
 130       Progress.OverallProgress(CurrentSize,TotalSize,FileSize,
 131                                string("Reading ") + Type() + " Indexes");
 132
 133       // Parse
 134       Progress.SubProgress(Pkg.Size());
 135       pkgTagSection Section;
 136       this->Section = &Section;
 137       string Prefix;
 138       unsigned long Hits = 0;
 139       unsigned long Chop = 0;
 140       while (Parser.Step(Section) == true)
 141       {
 142          Progress.Progress(Parser.Offset());
 143          string File;
 144          unsigned long Size;
 145          if (GetFile(File,Size) == false)
 146             return false;
 147
 148          if (Chop != 0)
 149             File = OrigPath + ChopDirs(File,Chop);
 150
 151          // See if the file exists
 152          bool Mangled = false;
 153          if (NoStat == false || Hits < 10)
 154          {
 155             // Attempt to fix broken structure
 156             if (Hits == 0)
 157             {
 158                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 159                    ReconstructChop(Chop,*I,File) == false)
 160                {
 161                   if (Debug == true)
 162                      clog << "Missed: " << File << endl;
 163                   NotFound++;
 164                   continue;
 165                }
 166                if (Chop != 0)
 167                   File = OrigPath + ChopDirs(File,Chop);
 168             }
 169
 170             // Get the size
 171             struct stat Buf;
 172             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 173                 Buf.st_size == 0)
 174             {
 175                // Attempt to fix busted symlink support for one instance
 176                string OrigFile = File;
 177                string::size_type Start = File.find("binary-");
 178                string::size_type End = File.find("/",Start+3);
 179                if (Start != string::npos && End != string::npos)
 180                {
 181                   File.replace(Start,End-Start,"binary-all");
 182                   Mangled = true;
 183                }
 184
 185                if (Mangled == false ||
 186                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 187                {
 188                   if (Debug == true)
 189                      clog << "Missed(2): " << OrigFile << endl;
 190                   NotFound++;
 191                   continue;
 192                }
 193             }
 194
 195             // Size match
 196             if ((unsigned)Buf.st_size != Size)
 197             {
 198                if (Debug == true)
 199                   clog << "Wrong Size: " << File << endl;
 200                WrongSize++;
 201                continue;
 202             }
 203          }
 204
 205          Packages++;
 206          Hits++;
 207
 208          // Copy it to the target package file
 209          if (Chop != 0 || Mangled == true)
 210          {
 211             if (RewriteEntry(Target,File) == false)
 212                continue;
 213          }
 214          else
 215          {
 216             const char *Start;
 217             const char *Stop;
 218             Section.GetSection(Start,Stop);
 219             if (Target.Write(Start,Stop-Start) == false)
 220                return false;
 221          }
 222       }
 223
 224       if (Debug == true)
 225          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 226
 227       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 228       {
 229          // Move out of the partial directory
 230          Target.Close();
 231          string FinalF = _config->FindDir("Dir::State::lists");
 232          FinalF += URItoFileName(S);
 233          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 234             return _error->Errno("rename","Failed to rename");
 235
 236          // Copy the release file
 237          sprintf(S,"cdrom:%s/%sRelease",Name.c_str(),(*I).c_str() + CDROM.length());
 238          string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 239          TargetF += URItoFileName(S);
 240          if (FileExists(*I + "Release") == true)
 241          {
 242             FileFd Target(TargetF,FileFd::WriteEmpty);
 243             FileFd Rel(*I + "Release",FileFd::ReadOnly);
 244             if (_error->PendingError() == true)
 245                return false;
 246
 247             if (CopyFile(Rel,Target) == false)
 248                return false;
 249          }
 250          else
 251          {
 252             // Empty release file
 253             FileFd Target(TargetF,FileFd::WriteEmpty);
 254          }
 255
 256          // Rename the release file
 257          FinalF = _config->FindDir("Dir::State::lists");
 258          FinalF += URItoFileName(S);
 259          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 260             return _error->Errno("rename","Failed to rename");
 261       }
 262
 263       /* Mangle the source to be in the proper notation with
 264          prefix dist [component] */
 265       *I = string(*I,Prefix.length());
 266       ConvertToSourceList(CDROM,*I);
 267       *I = Prefix + ' ' + *I;
 268
 269       CurrentSize += FileSize;
 270    }
 271    Progress.Done();
 272
 273    // Some stats
 274    cout << "Wrote " << Packages << " records" ;
 275    if (NotFound != 0)
 276       cout << " with " << NotFound << " missing files";
 277    if (NotFound != 0 && WrongSize != 0)
 278       cout << " and";
 279    if (WrongSize != 0)
 280       cout << " with " << WrongSize << " mismatched files";
 281    cout << '.' << endl;
 282
 283    if (Packages == 0)
 284       return _error->Warning("No valid records were found.");
 285
 286    if (NotFound + WrongSize > 10)
 287       cout << "Alot of entries were discarded, something may be wrong." << endl;
 288
 289    return true;
 290 }
 291                                                                         /*}}}*/
 292 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 293 // ---------------------------------------------------------------------
 294 /* */
 295 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 296 {
 297    string::size_type I = 0;
 298    do
 299    {
 300       I = Path.find('/',I+1);
 301       Depth--;
 302    }
 303    while (I != string::npos && Depth != 0);
 304
 305    if (I == string::npos)
 306       return string();
 307
 308    return string(Path,I+1);
 309 }
 310                                                                         /*}}}*/
 311 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* This prepends dir components from the path to the package files to
 314    the path to the deb until it is found */
 315 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 316                                   string File)
 317 {
 318    bool Debug = _config->FindB("Debug::aptcdrom",false);
 319    unsigned int Depth = 1;
 320    string MyPrefix = Prefix;
 321    while (1)
 322    {
 323       struct stat Buf;
 324       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 325       {
 326          if (Debug == true)
 327             cout << "Failed, " << CD + MyPrefix + File << endl;
 328          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 329             continue;
 330
 331          return false;
 332       }
 333       else
 334       {
 335          Prefix = MyPrefix;
 336          return true;
 337       }
 338    }
 339    return false;
 340 }
 341                                                                         /*}}}*/
 342 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 343 // ---------------------------------------------------------------------
 344 /* This removes path components from the filename and prepends the location
 345    of the package files until a file is found */
 346 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 347 {
 348    // Attempt to reconstruct the filename
 349    unsigned long Depth = 0;
 350    while (1)
 351    {
 352       struct stat Buf;
 353       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 354       {
 355          File = ChopDirs(File,1);
 356          Depth++;
 357          if (File.empty() == false)
 358             continue;
 359          return false;
 360       }
 361       else
 362       {
 363          Chop = Depth;
 364          return true;
 365       }
 366    }
 367    return false;
 368 }
 369                                                                         /*}}}*/
 370 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 371 // ---------------------------------------------------------------------
 372 /* We look for things in dists/ notation and convert them to
 373    <dist> <component> form otherwise it is left alone. This also strips
 374    the CD path. */
 375 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 376 {
 377    char S[300];
 378    sprintf(S,"binary-%s",_config->Find("Apt::Architecture").c_str());
 379
 380    // Strip the cdrom base path
 381    Path = string(Path,CD.length());
 382    if (Path.empty() == true)
 383       Path = "/";
 384
 385    // Too short to be a dists/ type
 386    if (Path.length() < strlen("dists/"))
 387       return;
 388
 389    // Not a dists type.
 390    if (stringcmp(Path.begin(),Path.begin()+strlen("dists/"),"dists/") != 0)
 391       return;
 392
 393    // Isolate the dist
 394    string::size_type Slash = strlen("dists/");
 395    string::size_type Slash2 = Path.find('/',Slash + 1);
 396    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 397       return;
 398    string Dist = string(Path,Slash,Slash2 - Slash);
 399
 400    // Isolate the component
 401    Slash = Path.find('/',Slash2+1);
 402    if (Slash == string::npos || Slash + 2 >= Path.length())
 403       return;
 404    string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 405
 406    // Verify the trailing binar - bit
 407    Slash2 = Path.find('/',Slash + 1);
 408    if (Slash == string::npos)
 409       return;
 410    string Binary = string(Path,Slash+1,Slash2 - Slash-1);
 411
 412    if (Binary != S && Binary != "source")
 413       return;
 414
 415    Path = Dist + ' ' + Comp;
 416 }
 417                                                                         /*}}}*/
 418 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 419 // ---------------------------------------------------------------------
 420 /* */
 421 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 422 {
 423    string::size_type I = 0;
 424    do
 425    {
 426       I = Path.find('/',I+1);
 427       Depth--;
 428    }
 429    while (I != string::npos && Depth != 0);
 430
 431    if (I == string::npos)
 432       return false;
 433
 434    To = string(Path,0,I+1);
 435    return true;
 436 }
 437                                                                         /*}}}*/
 438 // IndexCopy::CopyWithReplace - Copy a section and replace text         /*{{{*/
 439 // ---------------------------------------------------------------------
 440 /* */
 441 bool IndexCopy::CopyWithReplace(FileFd &Target,const char *Tag,string New)
 442 {
 443    // Mangle the output filename
 444    const char *Start;
 445    const char *Stop;
 446    const char *Filename;
 447    Section->Find(Tag,Filename,Stop);
 448
 449    /* We need to rewrite the filename field so we emit
 450       all fields except the filename file and rewrite that one */
 451    for (unsigned int I = 0; I != Section->Count(); I++)
 452    {
 453       Section->Get(Start,Stop,I);
 454       if (Start <= Filename && Stop > Filename)
 455       {
 456          char S[500];
 457          sprintf(S,"%s: %s\n",Tag,New.c_str());
 458          if (I + 1 == Section->Count())
 459             strcat(S,"\n");
 460          if (Target.Write(S,strlen(S)) == false)
 461             return false;
 462       }
 463       else
 464       {
 465          if (Target.Write(Start,Stop-Start) == false)
 466             return false;
 467          if (Stop[-1] != '\n')
 468             if (Target.Write("\n",1) == false)
 469                return false;
 470       }
 471    }
 472    if (Target.Write("\n",1) == false)
 473       return false;
 474 }
 475                                                                         /*}}}*/
 476 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 477 // ---------------------------------------------------------------------
 478 /* */
 479 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 480 {
 481    File = Section->FindS("Filename");
 482    Size = Section->FindI("Size");
 483    if (File.empty() || Size == 0)
 484       return _error->Error("Cannot find filename or size tag");
 485    return true;
 486 }
 487                                                                         /*}}}*/
 488 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 489 // ---------------------------------------------------------------------
 490 /* */
 491 bool PackageCopy::RewriteEntry(FileFd &Target,string File)
 492 {
 493    return CopyWithReplace(Target,"Filename",File);
 494 }
 495                                                                         /*}}}*/
 496 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* */
 499 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 500 {
 501    string Files = Section->FindS("Files");
 502    if (Files.empty() == true)
 503       return false;
 504
 505    // Stash the / terminated directory prefix
 506    string Base = Section->FindS("Directory");
 507    if (Base.empty() == false && Base[Base.length()-1] != '/')
 508       Base += '/';
 509
 510    // Iterate over the entire list grabbing each triplet
 511    const char *C = Files.c_str();
 512    string sSize;
 513    string MD5Hash;
 514
 515    // Parse each of the elements
 516    if (ParseQuoteWord(C,MD5Hash) == false ||
 517        ParseQuoteWord(C,sSize) == false ||
 518        ParseQuoteWord(C,File) == false)
 519       return _error->Error("Error parsing file record");
 520
 521    // Parse the size and append the directory
 522    Size = atoi(sSize.c_str());
 523    File = Base + File;
 524    return true;
 525 }
 526                                                                         /*}}}*/
 527 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 528 // ---------------------------------------------------------------------
 529 /* */
 530 bool SourceCopy::RewriteEntry(FileFd &Target,string File)
 531 {
 532    return CopyWithReplace(Target,"Directory",
 533                           string(File,0,File.rfind('/')));
 534 }
 535                                                                         /*}}}*/