cmdline/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.4 2000/01/16 05:36:17 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21
  22 #include <iostream.h>
  23 #include <unistd.h>
  24 #include <sys/stat.h>
  25 #include <stdio.h>
  26                                                                         /*}}}*/
  27
  28 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  29 // ---------------------------------------------------------------------
  30 /* */
  31 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List)
  32 {
  33    if (List.size() == 0)
  34       return true;
  35
  36    OpTextProgress Progress;
  37
  38    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  39    bool Debug = _config->FindB("Debug::aptcdrom",false);
  40
  41    // Prepare the progress indicator
  42    unsigned long TotalSize = 0;
  43    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  44    {
  45       struct stat Buf;
  46       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  47           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  48          return _error->Errno("stat","Stat failed for %s",
  49                               string(*I + GetFileName()).c_str());
  50       TotalSize += Buf.st_size;
  51    }
  52
  53    unsigned long CurrentSize = 0;
  54    unsigned int NotFound = 0;
  55    unsigned int WrongSize = 0;
  56    unsigned int Packages = 0;
  57    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  58    {
  59       string OrigPath = string(*I,CDROM.length());
  60       unsigned long FileSize = 0;
  61
  62       // Open the package file
  63       FileFd Pkg;
  64       if (FileExists(*I + GetFileName()) == true)
  65       {
  66          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  67          FileSize = Pkg.Size();
  68       }
  69       else
  70       {
  71          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  72          if (_error->PendingError() == true)
  73             return false;
  74          FileSize = From.Size();
  75
  76          // Get a temp file
  77          FILE *tmp = tmpfile();
  78          if (tmp == 0)
  79             return _error->Errno("tmpfile","Unable to create a tmp file");
  80          Pkg.Fd(dup(fileno(tmp)));
  81          fclose(tmp);
  82
  83          // Fork gzip
  84          int Process = fork();
  85          if (Process < 0)
  86             return _error->Errno("fork","Couldn't fork gzip");
  87
  88          // The child
  89          if (Process == 0)
  90          {
  91             dup2(From.Fd(),STDIN_FILENO);
  92             dup2(Pkg.Fd(),STDOUT_FILENO);
  93             SetCloseExec(STDIN_FILENO,false);
  94             SetCloseExec(STDOUT_FILENO,false);
  95
  96             const char *Args[3];
  97             Args[0] = _config->Find("Dir::bin::gzip","gzip").c_str();
  98             Args[1] = "-d";
  99             Args[2] = 0;
 100             execvp(Args[0],(char **)Args);
 101             exit(100);
 102          }
 103
 104          // Wait for gzip to finish
 105          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 106             return _error->Error("gzip failed, perhaps the disk is full.");
 107
 108          Pkg.Seek(0);
 109       }
 110       pkgTagFile Parser(Pkg);
 111       if (_error->PendingError() == true)
 112          return false;
 113
 114       // Open the output file
 115       char S[400];
 116       sprintf(S,"cdrom:[%s]/%s%s",Name.c_str(),(*I).c_str() + CDROM.length(),
 117               GetFileName());
 118       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 119       TargetF += URItoFileName(S);
 120       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 121          TargetF = "/dev/null";
 122       FileFd Target(TargetF,FileFd::WriteEmpty);
 123       if (_error->PendingError() == true)
 124          return false;
 125
 126       // Setup the progress meter
 127       Progress.OverallProgress(CurrentSize,TotalSize,FileSize,
 128                                string("Reading ") + Type() + " Indexes");
 129
 130       // Parse
 131       Progress.SubProgress(Pkg.Size());
 132       pkgTagSection Section;
 133       this->Section = &Section;
 134       string Prefix;
 135       unsigned long Hits = 0;
 136       unsigned long Chop = 0;
 137       while (Parser.Step(Section) == true)
 138       {
 139          Progress.Progress(Parser.Offset());
 140          string File;
 141          unsigned long Size;
 142          if (GetFile(File,Size) == false)
 143             return false;
 144
 145          if (Chop != 0)
 146             File = OrigPath + ChopDirs(File,Chop);
 147
 148          // See if the file exists
 149          bool Mangled = false;
 150          if (NoStat == false || Hits < 10)
 151          {
 152             // Attempt to fix broken structure
 153             if (Hits == 0)
 154             {
 155                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 156                    ReconstructChop(Chop,*I,File) == false)
 157                {
 158                   if (Debug == true)
 159                      clog << "Missed: " << File << endl;
 160                   NotFound++;
 161                   continue;
 162                }
 163                if (Chop != 0)
 164                   File = OrigPath + ChopDirs(File,Chop);
 165             }
 166
 167             // Get the size
 168             struct stat Buf;
 169             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 170                 Buf.st_size == 0)
 171             {
 172                // Attempt to fix busted symlink support for one instance
 173                string OrigFile = File;
 174                string::size_type Start = File.find("binary-");
 175                string::size_type End = File.find("/",Start+3);
 176                if (Start != string::npos && End != string::npos)
 177                {
 178                   File.replace(Start,End-Start,"binary-all");
 179                   Mangled = true;
 180                }
 181
 182                if (Mangled == false ||
 183                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 184                {
 185                   if (Debug == true)
 186                      clog << "Missed(2): " << OrigFile << endl;
 187                   NotFound++;
 188                   continue;
 189                }
 190             }
 191
 192             // Size match
 193             if ((unsigned)Buf.st_size != Size)
 194             {
 195                if (Debug == true)
 196                   clog << "Wrong Size: " << File << endl;
 197                WrongSize++;
 198                continue;
 199             }
 200          }
 201
 202          Packages++;
 203          Hits++;
 204
 205          // Copy it to the target package file
 206          if (Chop != 0 || Mangled == true)
 207          {
 208             if (RewriteEntry(Target,File) == false)
 209                continue;
 210          }
 211          else
 212          {
 213             const char *Start;
 214             const char *Stop;
 215             Section.GetSection(Start,Stop);
 216             if (Target.Write(Start,Stop-Start) == false)
 217                return false;
 218          }
 219       }
 220
 221       if (Debug == true)
 222          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 223
 224       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 225       {
 226          // Move out of the partial directory
 227          Target.Close();
 228          string FinalF = _config->FindDir("Dir::State::lists");
 229          FinalF += URItoFileName(S);
 230          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 231             return _error->Errno("rename","Failed to rename");
 232
 233          // Copy the release file
 234          sprintf(S,"cdrom:[%s]/%sRelease",Name.c_str(),(*I).c_str() + CDROM.length());
 235          string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 236          TargetF += URItoFileName(S);
 237          if (FileExists(*I + "Release") == true)
 238          {
 239             FileFd Target(TargetF,FileFd::WriteEmpty);
 240             FileFd Rel(*I + "Release",FileFd::ReadOnly);
 241             if (_error->PendingError() == true)
 242                return false;
 243
 244             if (CopyFile(Rel,Target) == false)
 245                return false;
 246          }
 247          else
 248          {
 249             // Empty release file
 250             FileFd Target(TargetF,FileFd::WriteEmpty);
 251          }
 252
 253          // Rename the release file
 254          FinalF = _config->FindDir("Dir::State::lists");
 255          FinalF += URItoFileName(S);
 256          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 257             return _error->Errno("rename","Failed to rename");
 258       }
 259
 260       /* Mangle the source to be in the proper notation with
 261          prefix dist [component] */
 262       *I = string(*I,Prefix.length());
 263       ConvertToSourceList(CDROM,*I);
 264       *I = Prefix + ' ' + *I;
 265
 266       CurrentSize += FileSize;
 267    }
 268    Progress.Done();
 269
 270    // Some stats
 271    cout << "Wrote " << Packages << " records" ;
 272    if (NotFound != 0)
 273       cout << " with " << NotFound << " missing files";
 274    if (NotFound != 0 && WrongSize != 0)
 275       cout << " and";
 276    if (WrongSize != 0)
 277       cout << " with " << WrongSize << " mismatched files";
 278    cout << '.' << endl;
 279
 280    if (Packages == 0)
 281       return _error->Warning("No valid records were found.");
 282
 283    if (NotFound + WrongSize > 10)
 284       cout << "Alot of entries were discarded, something may be wrong." << endl;
 285
 286    return true;
 287 }
 288                                                                         /*}}}*/
 289 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 290 // ---------------------------------------------------------------------
 291 /* */
 292 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 293 {
 294    string::size_type I = 0;
 295    do
 296    {
 297       I = Path.find('/',I+1);
 298       Depth--;
 299    }
 300    while (I != string::npos && Depth != 0);
 301
 302    if (I == string::npos)
 303       return string();
 304
 305    return string(Path,I+1);
 306 }
 307                                                                         /*}}}*/
 308 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 309 // ---------------------------------------------------------------------
 310 /* This prepends dir components from the path to the package files to
 311    the path to the deb until it is found */
 312 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 313                                   string File)
 314 {
 315    bool Debug = _config->FindB("Debug::aptcdrom",false);
 316    unsigned int Depth = 1;
 317    string MyPrefix = Prefix;
 318    while (1)
 319    {
 320       struct stat Buf;
 321       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 322       {
 323          if (Debug == true)
 324             cout << "Failed, " << CD + MyPrefix + File << endl;
 325          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 326             continue;
 327
 328          return false;
 329       }
 330       else
 331       {
 332          Prefix = MyPrefix;
 333          return true;
 334       }
 335    }
 336    return false;
 337 }
 338                                                                         /*}}}*/
 339 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 340 // ---------------------------------------------------------------------
 341 /* This removes path components from the filename and prepends the location
 342    of the package files until a file is found */
 343 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 344 {
 345    // Attempt to reconstruct the filename
 346    unsigned long Depth = 0;
 347    while (1)
 348    {
 349       struct stat Buf;
 350       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 351       {
 352          File = ChopDirs(File,1);
 353          Depth++;
 354          if (File.empty() == false)
 355             continue;
 356          return false;
 357       }
 358       else
 359       {
 360          Chop = Depth;
 361          return true;
 362       }
 363    }
 364    return false;
 365 }
 366                                                                         /*}}}*/
 367 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* We look for things in dists/ notation and convert them to
 370    <dist> <component> form otherwise it is left alone. This also strips
 371    the CD path. */
 372 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 373 {
 374    char S[300];
 375    sprintf(S,"binary-%s",_config->Find("Apt::Architecture").c_str());
 376
 377    // Strip the cdrom base path
 378    Path = string(Path,CD.length());
 379    if (Path.empty() == true)
 380       Path = "/";
 381
 382    // Too short to be a dists/ type
 383    if (Path.length() < strlen("dists/"))
 384       return;
 385
 386    // Not a dists type.
 387    if (stringcmp(Path.begin(),Path.begin()+strlen("dists/"),"dists/") != 0)
 388       return;
 389
 390    // Isolate the dist
 391    string::size_type Slash = strlen("dists/");
 392    string::size_type Slash2 = Path.find('/',Slash + 1);
 393    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 394       return;
 395    string Dist = string(Path,Slash,Slash2 - Slash);
 396
 397    // Isolate the component
 398    Slash = Path.find('/',Slash2+1);
 399    if (Slash == string::npos || Slash + 2 >= Path.length())
 400       return;
 401    string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 402
 403    // Verify the trailing binar - bit
 404    Slash2 = Path.find('/',Slash + 1);
 405    if (Slash == string::npos)
 406       return;
 407    string Binary = string(Path,Slash+1,Slash2 - Slash-1);
 408
 409    if (Binary != S && Binary != "source")
 410       return;
 411
 412    Path = Dist + ' ' + Comp;
 413 }
 414                                                                         /*}}}*/
 415 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 416 // ---------------------------------------------------------------------
 417 /* */
 418 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 419 {
 420    string::size_type I = 0;
 421    do
 422    {
 423       I = Path.find('/',I+1);
 424       Depth--;
 425    }
 426    while (I != string::npos && Depth != 0);
 427
 428    if (I == string::npos)
 429       return false;
 430
 431    To = string(Path,0,I+1);
 432    return true;
 433 }
 434                                                                         /*}}}*/
 435 // IndexCopy::CopyWithReplace - Copy a section and replace text         /*{{{*/
 436 // ---------------------------------------------------------------------
 437 /* */
 438 bool IndexCopy::CopyWithReplace(FileFd &Target,const char *Tag,string New)
 439 {
 440    // Mangle the output filename
 441    const char *Start;
 442    const char *Stop;
 443    const char *Filename;
 444    Section->Find(Tag,Filename,Stop);
 445
 446    /* We need to rewrite the filename field so we emit
 447       all fields except the filename file and rewrite that one */
 448    for (unsigned int I = 0; I != Section->Count(); I++)
 449    {
 450       Section->Get(Start,Stop,I);
 451       if (Start <= Filename && Stop > Filename)
 452       {
 453          char S[500];
 454          sprintf(S,"%s: %s\n",Tag,New.c_str());
 455          if (I + 1 == Section->Count())
 456             strcat(S,"\n");
 457          if (Target.Write(S,strlen(S)) == false)
 458             return false;
 459       }
 460       else
 461       {
 462          if (Target.Write(Start,Stop-Start) == false)
 463             return false;
 464          if (Stop[-1] != '\n')
 465             if (Target.Write("\n",1) == false)
 466                return false;
 467       }
 468    }
 469    if (Target.Write("\n",1) == false)
 470       return false;
 471 }
 472                                                                         /*}}}*/
 473 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 474 // ---------------------------------------------------------------------
 475 /* */
 476 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 477 {
 478    File = Section->FindS("Filename");
 479    Size = Section->FindI("Size");
 480    if (File.empty() || Size == 0)
 481       return _error->Error("Cannot find filename or size tag");
 482    return true;
 483 }
 484                                                                         /*}}}*/
 485 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 486 // ---------------------------------------------------------------------
 487 /* */
 488 bool PackageCopy::RewriteEntry(FileFd &Target,string File)
 489 {
 490    return CopyWithReplace(Target,"Filename",File);
 491 }
 492                                                                         /*}}}*/
 493 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 494 // ---------------------------------------------------------------------
 495 /* */
 496 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 497 {
 498    string Files = Section->FindS("Files");
 499    if (Files.empty() == true)
 500       return false;
 501
 502    // Stash the / terminated directory prefix
 503    string Base = Section->FindS("Directory");
 504    if (Base.empty() == false && Base[Base.length()-1] != '/')
 505       Base += '/';
 506
 507    // Iterate over the entire list grabbing each triplet
 508    const char *C = Files.c_str();
 509    string sSize;
 510    string MD5Hash;
 511
 512    // Parse each of the elements
 513    if (ParseQuoteWord(C,MD5Hash) == false ||
 514        ParseQuoteWord(C,sSize) == false ||
 515        ParseQuoteWord(C,File) == false)
 516       return _error->Error("Error parsing file record");
 517
 518    // Parse the size and append the directory
 519    Size = atoi(sSize.c_str());
 520    File = Base + File;
 521    return true;
 522 }
 523                                                                         /*}}}*/
 524 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 525 // ---------------------------------------------------------------------
 526 /* */
 527 bool SourceCopy::RewriteEntry(FileFd &Target,string File)
 528 {
 529    return CopyWithReplace(Target,"Directory",
 530                           string(File,0,File.rfind('/')));
 531 }
 532                                                                         /*}}}*/