cmdline/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.6 2001/02/20 07:03:17 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21
  22 #include <iostream.h>
  23 #include <unistd.h>
  24 #include <sys/stat.h>
  25 #include <stdio.h>
  26                                                                         /*}}}*/
  27
  28 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  29 // ---------------------------------------------------------------------
  30 /* */
  31 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List)
  32 {
  33    if (List.size() == 0)
  34       return true;
  35
  36    OpTextProgress Progress;
  37
  38    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  39    bool Debug = _config->FindB("Debug::aptcdrom",false);
  40
  41    // Prepare the progress indicator
  42    unsigned long TotalSize = 0;
  43    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  44    {
  45       struct stat Buf;
  46       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  47           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  48          return _error->Errno("stat","Stat failed for %s",
  49                               string(*I + GetFileName()).c_str());
  50       TotalSize += Buf.st_size;
  51    }
  52
  53    unsigned long CurrentSize = 0;
  54    unsigned int NotFound = 0;
  55    unsigned int WrongSize = 0;
  56    unsigned int Packages = 0;
  57    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  58    {
  59       string OrigPath = string(*I,CDROM.length());
  60       unsigned long FileSize = 0;
  61
  62       // Open the package file
  63       FileFd Pkg;
  64       if (FileExists(*I + GetFileName()) == true)
  65       {
  66          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  67          FileSize = Pkg.Size();
  68       }
  69       else
  70       {
  71          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  72          if (_error->PendingError() == true)
  73             return false;
  74          FileSize = From.Size();
  75
  76          // Get a temp file
  77          FILE *tmp = tmpfile();
  78          if (tmp == 0)
  79             return _error->Errno("tmpfile","Unable to create a tmp file");
  80          Pkg.Fd(dup(fileno(tmp)));
  81          fclose(tmp);
  82
  83          // Fork gzip
  84          int Process = fork();
  85          if (Process < 0)
  86             return _error->Errno("fork","Couldn't fork gzip");
  87
  88          // The child
  89          if (Process == 0)
  90          {
  91             dup2(From.Fd(),STDIN_FILENO);
  92             dup2(Pkg.Fd(),STDOUT_FILENO);
  93             SetCloseExec(STDIN_FILENO,false);
  94             SetCloseExec(STDOUT_FILENO,false);
  95
  96             const char *Args[3];
  97             Args[0] = _config->Find("Dir::bin::gzip","gzip").c_str();
  98             Args[1] = "-d";
  99             Args[2] = 0;
 100             execvp(Args[0],(char **)Args);
 101             exit(100);
 102          }
 103
 104          // Wait for gzip to finish
 105          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 106             return _error->Error("gzip failed, perhaps the disk is full.");
 107
 108          Pkg.Seek(0);
 109       }
 110       pkgTagFile Parser(&Pkg);
 111       if (_error->PendingError() == true)
 112          return false;
 113
 114       // Open the output file
 115       char S[400];
 116       sprintf(S,"cdrom:[%s]/%s%s",Name.c_str(),(*I).c_str() + CDROM.length(),
 117               GetFileName());
 118       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 119       TargetF += URItoFileName(S);
 120       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 121          TargetF = "/dev/null";
 122       FileFd Target(TargetF,FileFd::WriteEmpty);
 123       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 124       if (_error->PendingError() == true)
 125          return false;
 126       if (TargetFl == 0)
 127          return _error->Errno("fdopen","Failed to reopen fd");
 128
 129       // Setup the progress meter
 130       Progress.OverallProgress(CurrentSize,TotalSize,FileSize,
 131                                string("Reading ") + Type() + " Indexes");
 132
 133       // Parse
 134       Progress.SubProgress(Pkg.Size());
 135       pkgTagSection Section;
 136       this->Section = &Section;
 137       string Prefix;
 138       unsigned long Hits = 0;
 139       unsigned long Chop = 0;
 140       while (Parser.Step(Section) == true)
 141       {
 142          Progress.Progress(Parser.Offset());
 143          string File;
 144          unsigned long Size;
 145          if (GetFile(File,Size) == false)
 146          {
 147             fclose(TargetFl);
 148             return false;
 149          }
 150
 151          if (Chop != 0)
 152             File = OrigPath + ChopDirs(File,Chop);
 153
 154          // See if the file exists
 155          bool Mangled = false;
 156          if (NoStat == false || Hits < 10)
 157          {
 158             // Attempt to fix broken structure
 159             if (Hits == 0)
 160             {
 161                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 162                    ReconstructChop(Chop,*I,File) == false)
 163                {
 164                   if (Debug == true)
 165                      clog << "Missed: " << File << endl;
 166                   NotFound++;
 167                   continue;
 168                }
 169                if (Chop != 0)
 170                   File = OrigPath + ChopDirs(File,Chop);
 171             }
 172
 173             // Get the size
 174             struct stat Buf;
 175             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 176                 Buf.st_size == 0)
 177             {
 178                // Attempt to fix busted symlink support for one instance
 179                string OrigFile = File;
 180                string::size_type Start = File.find("binary-");
 181                string::size_type End = File.find("/",Start+3);
 182                if (Start != string::npos && End != string::npos)
 183                {
 184                   File.replace(Start,End-Start,"binary-all");
 185                   Mangled = true;
 186                }
 187
 188                if (Mangled == false ||
 189                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 190                {
 191                   if (Debug == true)
 192                      clog << "Missed(2): " << OrigFile << endl;
 193                   NotFound++;
 194                   continue;
 195                }
 196             }
 197
 198             // Size match
 199             if ((unsigned)Buf.st_size != Size)
 200             {
 201                if (Debug == true)
 202                   clog << "Wrong Size: " << File << endl;
 203                WrongSize++;
 204                continue;
 205             }
 206          }
 207
 208          Packages++;
 209          Hits++;
 210
 211          if (RewriteEntry(TargetFl,File) == false)
 212          {
 213             fclose(TargetFl);
 214             return false;
 215          }
 216       }
 217       fclose(TargetFl);
 218
 219       if (Debug == true)
 220          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 221
 222       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 223       {
 224          // Move out of the partial directory
 225          Target.Close();
 226          string FinalF = _config->FindDir("Dir::State::lists");
 227          FinalF += URItoFileName(S);
 228          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 229             return _error->Errno("rename","Failed to rename");
 230
 231          // Copy the release file
 232          sprintf(S,"cdrom:[%s]/%sRelease",Name.c_str(),(*I).c_str() + CDROM.length());
 233          string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 234          TargetF += URItoFileName(S);
 235          if (FileExists(*I + "Release") == true)
 236          {
 237             FileFd Target(TargetF,FileFd::WriteEmpty);
 238             FileFd Rel(*I + "Release",FileFd::ReadOnly);
 239             if (_error->PendingError() == true)
 240                return false;
 241
 242             if (CopyFile(Rel,Target) == false)
 243                return false;
 244          }
 245          else
 246          {
 247             // Empty release file
 248             FileFd Target(TargetF,FileFd::WriteEmpty);
 249          }
 250
 251          // Rename the release file
 252          FinalF = _config->FindDir("Dir::State::lists");
 253          FinalF += URItoFileName(S);
 254          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 255             return _error->Errno("rename","Failed to rename");
 256       }
 257
 258       /* Mangle the source to be in the proper notation with
 259          prefix dist [component] */
 260       *I = string(*I,Prefix.length());
 261       ConvertToSourceList(CDROM,*I);
 262       *I = Prefix + ' ' + *I;
 263
 264       CurrentSize += FileSize;
 265    }
 266    Progress.Done();
 267
 268    // Some stats
 269    cout << "Wrote " << Packages << " records" ;
 270    if (NotFound != 0)
 271       cout << " with " << NotFound << " missing files";
 272    if (NotFound != 0 && WrongSize != 0)
 273       cout << " and";
 274    if (WrongSize != 0)
 275       cout << " with " << WrongSize << " mismatched files";
 276    cout << '.' << endl;
 277
 278    if (Packages == 0)
 279       return _error->Warning("No valid records were found.");
 280
 281    if (NotFound + WrongSize > 10)
 282       cout << "Alot of entries were discarded, something may be wrong." << endl;
 283
 284    return true;
 285 }
 286                                                                         /*}}}*/
 287 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 288 // ---------------------------------------------------------------------
 289 /* */
 290 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 291 {
 292    string::size_type I = 0;
 293    do
 294    {
 295       I = Path.find('/',I+1);
 296       Depth--;
 297    }
 298    while (I != string::npos && Depth != 0);
 299
 300    if (I == string::npos)
 301       return string();
 302
 303    return string(Path,I+1);
 304 }
 305                                                                         /*}}}*/
 306 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 307 // ---------------------------------------------------------------------
 308 /* This prepends dir components from the path to the package files to
 309    the path to the deb until it is found */
 310 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 311                                   string File)
 312 {
 313    bool Debug = _config->FindB("Debug::aptcdrom",false);
 314    unsigned int Depth = 1;
 315    string MyPrefix = Prefix;
 316    while (1)
 317    {
 318       struct stat Buf;
 319       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 320       {
 321          if (Debug == true)
 322             cout << "Failed, " << CD + MyPrefix + File << endl;
 323          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 324             continue;
 325
 326          return false;
 327       }
 328       else
 329       {
 330          Prefix = MyPrefix;
 331          return true;
 332       }
 333    }
 334    return false;
 335 }
 336                                                                         /*}}}*/
 337 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* This removes path components from the filename and prepends the location
 340    of the package files until a file is found */
 341 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 342 {
 343    // Attempt to reconstruct the filename
 344    unsigned long Depth = 0;
 345    while (1)
 346    {
 347       struct stat Buf;
 348       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 349       {
 350          File = ChopDirs(File,1);
 351          Depth++;
 352          if (File.empty() == false)
 353             continue;
 354          return false;
 355       }
 356       else
 357       {
 358          Chop = Depth;
 359          return true;
 360       }
 361    }
 362    return false;
 363 }
 364                                                                         /*}}}*/
 365 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 366 // ---------------------------------------------------------------------
 367 /* We look for things in dists/ notation and convert them to
 368    <dist> <component> form otherwise it is left alone. This also strips
 369    the CD path.
 370
 371    This implements a regex sort of like:
 372     (.*)/dists/([^/]*)/(.*)/binary-*
 373      ^          ^      ^- Component
 374      |          |-------- Distribution
 375      |------------------- Path
 376
 377    It was deciced to use only a single word for dist (rather than say
 378    unstable/non-us) to increase the chance that each CD gets a single
 379    line in sources.list.
 380  */
 381 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 382 {
 383    char S[300];
 384    sprintf(S,"binary-%s",_config->Find("Apt::Architecture").c_str());
 385
 386    // Strip the cdrom base path
 387    Path = string(Path,CD.length());
 388    if (Path.empty() == true)
 389       Path = "/";
 390
 391    // Too short to be a dists/ type
 392    if (Path.length() < strlen("dists/"))
 393       return;
 394
 395    // Not a dists type.
 396    if (stringcmp(Path.begin(),Path.begin()+strlen("dists/"),"dists/") != 0)
 397       return;
 398
 399    // Isolate the dist
 400    string::size_type Slash = strlen("dists/");
 401    string::size_type Slash2 = Path.find('/',Slash + 1);
 402    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 403       return;
 404    string Dist = string(Path,Slash,Slash2 - Slash);
 405
 406    // Isolate the component
 407    Slash = Slash2;
 408    for (unsigned I = 0; I != 10; I++)
 409    {
 410       Slash = Path.find('/',Slash+1);
 411       if (Slash == string::npos || Slash + 2 >= Path.length())
 412          return;
 413       string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 414
 415       // Verify the trailing binary- bit
 416       string::size_type BinSlash = Path.find('/',Slash + 1);
 417       if (Slash == string::npos)
 418          return;
 419       string Binary = string(Path,Slash+1,BinSlash - Slash-1);
 420
 421       if (Binary != S && Binary != "source")
 422          continue;
 423
 424       Path = Dist + ' ' + Comp;
 425       return;
 426    }
 427 }
 428                                                                         /*}}}*/
 429 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 430 // ---------------------------------------------------------------------
 431 /* */
 432 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 433 {
 434    string::size_type I = 0;
 435    do
 436    {
 437       I = Path.find('/',I+1);
 438       Depth--;
 439    }
 440    while (I != string::npos && Depth != 0);
 441
 442    if (I == string::npos)
 443       return false;
 444
 445    To = string(Path,0,I+1);
 446    return true;
 447 }
 448                                                                         /*}}}*/
 449 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 450 // ---------------------------------------------------------------------
 451 /* */
 452 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 453 {
 454    File = Section->FindS("Filename");
 455    Size = Section->FindI("Size");
 456    if (File.empty() || Size == 0)
 457       return _error->Error("Cannot find filename or size tag");
 458    return true;
 459 }
 460                                                                         /*}}}*/
 461 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 462 // ---------------------------------------------------------------------
 463 /* */
 464 bool PackageCopy::RewriteEntry(FILE *Target,string File)
 465 {
 466    TFRewriteData Changes[] = {{"Filename",File.c_str()},
 467                               {}};
 468
 469    if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
 470       return false;
 471    fputc('\n',Target);
 472    return true;
 473 }
 474                                                                         /*}}}*/
 475 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 476 // ---------------------------------------------------------------------
 477 /* */
 478 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 479 {
 480    string Files = Section->FindS("Files");
 481    if (Files.empty() == true)
 482       return false;
 483
 484    // Stash the / terminated directory prefix
 485    string Base = Section->FindS("Directory");
 486    if (Base.empty() == false && Base[Base.length()-1] != '/')
 487       Base += '/';
 488
 489    // Read the first file triplet
 490    const char *C = Files.c_str();
 491    string sSize;
 492    string MD5Hash;
 493
 494    // Parse each of the elements
 495    if (ParseQuoteWord(C,MD5Hash) == false ||
 496        ParseQuoteWord(C,sSize) == false ||
 497        ParseQuoteWord(C,File) == false)
 498       return _error->Error("Error parsing file record");
 499
 500    // Parse the size and append the directory
 501    Size = atoi(sSize.c_str());
 502    File = Base + File;
 503    return true;
 504 }
 505                                                                         /*}}}*/
 506 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 507 // ---------------------------------------------------------------------
 508 /* */
 509 bool SourceCopy::RewriteEntry(FILE *Target,string File)
 510 {
 511    string Dir(File,0,File.rfind('/'));
 512    TFRewriteData Changes[] = {{"Directory",Dir.c_str()},
 513                               {}};
 514
 515    if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
 516       return false;
 517    fputc('\n',Target);
 518    return true;
 519 }
 520                                                                         /*}}}*/