cmdline/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.10 2002/03/26 07:38:58 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21
  22 #include <iostream>
  23 #include <unistd.h>
  24 #include <sys/stat.h>
  25 #include <stdio.h>
  26                                                                         /*}}}*/
  27
  28 using namespace std;
  29
  30 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  31 // ---------------------------------------------------------------------
  32 /* */
  33 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List)
  34 {
  35    if (List.size() == 0)
  36       return true;
  37
  38    OpTextProgress Progress;
  39
  40    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  41    bool Debug = _config->FindB("Debug::aptcdrom",false);
  42
  43    // Prepare the progress indicator
  44    unsigned long TotalSize = 0;
  45    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  46    {
  47       struct stat Buf;
  48       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  49           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  50          return _error->Errno("stat","Stat failed for %s",
  51                               string(*I + GetFileName()).c_str());
  52       TotalSize += Buf.st_size;
  53    }
  54
  55    unsigned long CurrentSize = 0;
  56    unsigned int NotFound = 0;
  57    unsigned int WrongSize = 0;
  58    unsigned int Packages = 0;
  59    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  60    {
  61       string OrigPath = string(*I,CDROM.length());
  62       unsigned long FileSize = 0;
  63
  64       // Open the package file
  65       FileFd Pkg;
  66       if (FileExists(*I + GetFileName()) == true)
  67       {
  68          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  69          FileSize = Pkg.Size();
  70       }
  71       else
  72       {
  73          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  74          if (_error->PendingError() == true)
  75             return false;
  76          FileSize = From.Size();
  77
  78          // Get a temp file
  79          FILE *tmp = tmpfile();
  80          if (tmp == 0)
  81             return _error->Errno("tmpfile","Unable to create a tmp file");
  82          Pkg.Fd(dup(fileno(tmp)));
  83          fclose(tmp);
  84
  85          // Fork gzip
  86          int Process = fork();
  87          if (Process < 0)
  88             return _error->Errno("fork","Couldn't fork gzip");
  89
  90          // The child
  91          if (Process == 0)
  92          {
  93             dup2(From.Fd(),STDIN_FILENO);
  94             dup2(Pkg.Fd(),STDOUT_FILENO);
  95             SetCloseExec(STDIN_FILENO,false);
  96             SetCloseExec(STDOUT_FILENO,false);
  97
  98             const char *Args[3];
  99             string Tmp =  _config->Find("Dir::bin::gzip","gzip");
 100             Args[0] = Tmp.c_str();
 101             Args[1] = "-d";
 102             Args[2] = 0;
 103             execvp(Args[0],(char **)Args);
 104             exit(100);
 105          }
 106
 107          // Wait for gzip to finish
 108          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 109             return _error->Error("gzip failed, perhaps the disk is full.");
 110
 111          Pkg.Seek(0);
 112       }
 113       pkgTagFile Parser(&Pkg);
 114       if (_error->PendingError() == true)
 115          return false;
 116
 117       // Open the output file
 118       char S[400];
 119       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",Name.c_str(),
 120                (*I).c_str() + CDROM.length(),GetFileName());
 121       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 122       TargetF += URItoFileName(S);
 123       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 124          TargetF = "/dev/null";
 125       FileFd Target(TargetF,FileFd::WriteEmpty);
 126       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 127       if (_error->PendingError() == true)
 128          return false;
 129       if (TargetFl == 0)
 130          return _error->Errno("fdopen","Failed to reopen fd");
 131
 132       // Setup the progress meter
 133       Progress.OverallProgress(CurrentSize,TotalSize,FileSize,
 134                                string("Reading ") + Type() + " Indexes");
 135
 136       // Parse
 137       Progress.SubProgress(Pkg.Size());
 138       pkgTagSection Section;
 139       this->Section = &Section;
 140       string Prefix;
 141       unsigned long Hits = 0;
 142       unsigned long Chop = 0;
 143       while (Parser.Step(Section) == true)
 144       {
 145          Progress.Progress(Parser.Offset());
 146          string File;
 147          unsigned long Size;
 148          if (GetFile(File,Size) == false)
 149          {
 150             fclose(TargetFl);
 151             return false;
 152          }
 153
 154          if (Chop != 0)
 155             File = OrigPath + ChopDirs(File,Chop);
 156
 157          // See if the file exists
 158          bool Mangled = false;
 159          if (NoStat == false || Hits < 10)
 160          {
 161             // Attempt to fix broken structure
 162             if (Hits == 0)
 163             {
 164                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 165                    ReconstructChop(Chop,*I,File) == false)
 166                {
 167                   if (Debug == true)
 168                      clog << "Missed: " << File << endl;
 169                   NotFound++;
 170                   continue;
 171                }
 172                if (Chop != 0)
 173                   File = OrigPath + ChopDirs(File,Chop);
 174             }
 175
 176             // Get the size
 177             struct stat Buf;
 178             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 179                 Buf.st_size == 0)
 180             {
 181                // Attempt to fix busted symlink support for one instance
 182                string OrigFile = File;
 183                string::size_type Start = File.find("binary-");
 184                string::size_type End = File.find("/",Start+3);
 185                if (Start != string::npos && End != string::npos)
 186                {
 187                   File.replace(Start,End-Start,"binary-all");
 188                   Mangled = true;
 189                }
 190
 191                if (Mangled == false ||
 192                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 193                {
 194                   if (Debug == true)
 195                      clog << "Missed(2): " << OrigFile << endl;
 196                   NotFound++;
 197                   continue;
 198                }
 199             }
 200
 201             // Size match
 202             if ((unsigned)Buf.st_size != Size)
 203             {
 204                if (Debug == true)
 205                   clog << "Wrong Size: " << File << endl;
 206                WrongSize++;
 207                continue;
 208             }
 209          }
 210
 211          Packages++;
 212          Hits++;
 213
 214          if (RewriteEntry(TargetFl,File) == false)
 215          {
 216             fclose(TargetFl);
 217             return false;
 218          }
 219       }
 220       fclose(TargetFl);
 221
 222       if (Debug == true)
 223          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 224
 225       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 226       {
 227          // Move out of the partial directory
 228          Target.Close();
 229          string FinalF = _config->FindDir("Dir::State::lists");
 230          FinalF += URItoFileName(S);
 231          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 232             return _error->Errno("rename","Failed to rename");
 233
 234          // Copy the release file
 235          snprintf(S,sizeof(S),"cdrom:[%s]/%sRelease",Name.c_str(),
 236                   (*I).c_str() + CDROM.length());
 237          string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 238          TargetF += URItoFileName(S);
 239          if (FileExists(*I + "Release") == true)
 240          {
 241             FileFd Target(TargetF,FileFd::WriteEmpty);
 242             FileFd Rel(*I + "Release",FileFd::ReadOnly);
 243             if (_error->PendingError() == true)
 244                return false;
 245
 246             if (CopyFile(Rel,Target) == false)
 247                return false;
 248          }
 249          else
 250          {
 251             // Empty release file
 252             FileFd Target(TargetF,FileFd::WriteEmpty);
 253          }
 254
 255          // Rename the release file
 256          FinalF = _config->FindDir("Dir::State::lists");
 257          FinalF += URItoFileName(S);
 258          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 259             return _error->Errno("rename","Failed to rename");
 260       }
 261
 262       /* Mangle the source to be in the proper notation with
 263          prefix dist [component] */
 264       *I = string(*I,Prefix.length());
 265       ConvertToSourceList(CDROM,*I);
 266       *I = Prefix + ' ' + *I;
 267
 268       CurrentSize += FileSize;
 269    }
 270    Progress.Done();
 271
 272    // Some stats
 273    cout << "Wrote " << Packages << " records" ;
 274    if (NotFound != 0)
 275       cout << " with " << NotFound << " missing files";
 276    if (NotFound != 0 && WrongSize != 0)
 277       cout << " and";
 278    if (WrongSize != 0)
 279       cout << " with " << WrongSize << " mismatched files";
 280    cout << '.' << endl;
 281
 282    if (Packages == 0)
 283       _error->Warning("No valid records were found.");
 284
 285    if (NotFound + WrongSize > 10)
 286       cout << "Alot of entries were discarded, something may be wrong." << endl;
 287
 288    return true;
 289 }
 290                                                                         /*}}}*/
 291 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 292 // ---------------------------------------------------------------------
 293 /* */
 294 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 295 {
 296    string::size_type I = 0;
 297    do
 298    {
 299       I = Path.find('/',I+1);
 300       Depth--;
 301    }
 302    while (I != string::npos && Depth != 0);
 303
 304    if (I == string::npos)
 305       return string();
 306
 307    return string(Path,I+1);
 308 }
 309                                                                         /*}}}*/
 310 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 311 // ---------------------------------------------------------------------
 312 /* This prepends dir components from the path to the package files to
 313    the path to the deb until it is found */
 314 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 315                                   string File)
 316 {
 317    bool Debug = _config->FindB("Debug::aptcdrom",false);
 318    unsigned int Depth = 1;
 319    string MyPrefix = Prefix;
 320    while (1)
 321    {
 322       struct stat Buf;
 323       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 324       {
 325          if (Debug == true)
 326             cout << "Failed, " << CD + MyPrefix + File << endl;
 327          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 328             continue;
 329
 330          return false;
 331       }
 332       else
 333       {
 334          Prefix = MyPrefix;
 335          return true;
 336       }
 337    }
 338    return false;
 339 }
 340                                                                         /*}}}*/
 341 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* This removes path components from the filename and prepends the location
 344    of the package files until a file is found */
 345 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 346 {
 347    // Attempt to reconstruct the filename
 348    unsigned long Depth = 0;
 349    while (1)
 350    {
 351       struct stat Buf;
 352       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 353       {
 354          File = ChopDirs(File,1);
 355          Depth++;
 356          if (File.empty() == false)
 357             continue;
 358          return false;
 359       }
 360       else
 361       {
 362          Chop = Depth;
 363          return true;
 364       }
 365    }
 366    return false;
 367 }
 368                                                                         /*}}}*/
 369 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 370 // ---------------------------------------------------------------------
 371 /* We look for things in dists/ notation and convert them to
 372    <dist> <component> form otherwise it is left alone. This also strips
 373    the CD path.
 374
 375    This implements a regex sort of like:
 376     (.*)/dists/([^/]*)/(.*)/binary-*
 377      ^          ^      ^- Component
 378      |          |-------- Distribution
 379      |------------------- Path
 380
 381    It was deciced to use only a single word for dist (rather than say
 382    unstable/non-us) to increase the chance that each CD gets a single
 383    line in sources.list.
 384  */
 385 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 386 {
 387    char S[300];
 388    snprintf(S,sizeof(S),"binary-%s",_config->Find("Apt::Architecture").c_str());
 389
 390    // Strip the cdrom base path
 391    Path = string(Path,CD.length());
 392    if (Path.empty() == true)
 393       Path = "/";
 394
 395    // Too short to be a dists/ type
 396    if (Path.length() < strlen("dists/"))
 397       return;
 398
 399    // Not a dists type.
 400    if (stringcmp(Path.c_str(),Path.c_str()+strlen("dists/"),"dists/") != 0)
 401       return;
 402
 403    // Isolate the dist
 404    string::size_type Slash = strlen("dists/");
 405    string::size_type Slash2 = Path.find('/',Slash + 1);
 406    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 407       return;
 408    string Dist = string(Path,Slash,Slash2 - Slash);
 409
 410    // Isolate the component
 411    Slash = Slash2;
 412    for (unsigned I = 0; I != 10; I++)
 413    {
 414       Slash = Path.find('/',Slash+1);
 415       if (Slash == string::npos || Slash + 2 >= Path.length())
 416          return;
 417       string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 418
 419       // Verify the trailing binary- bit
 420       string::size_type BinSlash = Path.find('/',Slash + 1);
 421       if (Slash == string::npos)
 422          return;
 423       string Binary = string(Path,Slash+1,BinSlash - Slash-1);
 424
 425       if (Binary != S && Binary != "source")
 426          continue;
 427
 428       Path = Dist + ' ' + Comp;
 429       return;
 430    }
 431 }
 432                                                                         /*}}}*/
 433 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 434 // ---------------------------------------------------------------------
 435 /* */
 436 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 437 {
 438    string::size_type I = 0;
 439    do
 440    {
 441       I = Path.find('/',I+1);
 442       Depth--;
 443    }
 444    while (I != string::npos && Depth != 0);
 445
 446    if (I == string::npos)
 447       return false;
 448
 449    To = string(Path,0,I+1);
 450    return true;
 451 }
 452                                                                         /*}}}*/
 453 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 454 // ---------------------------------------------------------------------
 455 /* */
 456 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 457 {
 458    File = Section->FindS("Filename");
 459    Size = Section->FindI("Size");
 460    if (File.empty() || Size == 0)
 461       return _error->Error("Cannot find filename or size tag");
 462    return true;
 463 }
 464                                                                         /*}}}*/
 465 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 466 // ---------------------------------------------------------------------
 467 /* */
 468 bool PackageCopy::RewriteEntry(FILE *Target,string File)
 469 {
 470    TFRewriteData Changes[] = {{"Filename",File.c_str()},
 471                               {}};
 472
 473    if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
 474       return false;
 475    fputc('\n',Target);
 476    return true;
 477 }
 478                                                                         /*}}}*/
 479 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 480 // ---------------------------------------------------------------------
 481 /* */
 482 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 483 {
 484    string Files = Section->FindS("Files");
 485    if (Files.empty() == true)
 486       return false;
 487
 488    // Stash the / terminated directory prefix
 489    string Base = Section->FindS("Directory");
 490    if (Base.empty() == false && Base[Base.length()-1] != '/')
 491       Base += '/';
 492
 493    // Read the first file triplet
 494    const char *C = Files.c_str();
 495    string sSize;
 496    string MD5Hash;
 497
 498    // Parse each of the elements
 499    if (ParseQuoteWord(C,MD5Hash) == false ||
 500        ParseQuoteWord(C,sSize) == false ||
 501        ParseQuoteWord(C,File) == false)
 502       return _error->Error("Error parsing file record");
 503
 504    // Parse the size and append the directory
 505    Size = atoi(sSize.c_str());
 506    File = Base + File;
 507    return true;
 508 }
 509                                                                         /*}}}*/
 510 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 511 // ---------------------------------------------------------------------
 512 /* */
 513 bool SourceCopy::RewriteEntry(FILE *Target,string File)
 514 {
 515    string Dir(File,0,File.rfind('/'));
 516    TFRewriteData Changes[] = {{"Directory",Dir.c_str()},
 517                               {}};
 518
 519    if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
 520       return false;
 521    fputc('\n',Target);
 522    return true;
 523 }
 524                                                                         /*}}}*/