cmdline/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.7 2001/03/13 05:23:42 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21
  22 #include <iostream.h>
  23 #include <unistd.h>
  24 #include <sys/stat.h>
  25 #include <stdio.h>
  26                                                                         /*}}}*/
  27
  28 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  29 // ---------------------------------------------------------------------
  30 /* */
  31 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List)
  32 {
  33    if (List.size() == 0)
  34       return true;
  35
  36    OpTextProgress Progress;
  37
  38    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  39    bool Debug = _config->FindB("Debug::aptcdrom",false);
  40
  41    // Prepare the progress indicator
  42    unsigned long TotalSize = 0;
  43    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  44    {
  45       struct stat Buf;
  46       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  47           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  48          return _error->Errno("stat","Stat failed for %s",
  49                               string(*I + GetFileName()).c_str());
  50       TotalSize += Buf.st_size;
  51    }
  52
  53    unsigned long CurrentSize = 0;
  54    unsigned int NotFound = 0;
  55    unsigned int WrongSize = 0;
  56    unsigned int Packages = 0;
  57    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  58    {
  59       string OrigPath = string(*I,CDROM.length());
  60       unsigned long FileSize = 0;
  61
  62       // Open the package file
  63       FileFd Pkg;
  64       if (FileExists(*I + GetFileName()) == true)
  65       {
  66          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  67          FileSize = Pkg.Size();
  68       }
  69       else
  70       {
  71          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  72          if (_error->PendingError() == true)
  73             return false;
  74          FileSize = From.Size();
  75
  76          // Get a temp file
  77          FILE *tmp = tmpfile();
  78          if (tmp == 0)
  79             return _error->Errno("tmpfile","Unable to create a tmp file");
  80          Pkg.Fd(dup(fileno(tmp)));
  81          fclose(tmp);
  82
  83          // Fork gzip
  84          int Process = fork();
  85          if (Process < 0)
  86             return _error->Errno("fork","Couldn't fork gzip");
  87
  88          // The child
  89          if (Process == 0)
  90          {
  91             dup2(From.Fd(),STDIN_FILENO);
  92             dup2(Pkg.Fd(),STDOUT_FILENO);
  93             SetCloseExec(STDIN_FILENO,false);
  94             SetCloseExec(STDOUT_FILENO,false);
  95
  96             const char *Args[3];
  97             Args[0] = _config->Find("Dir::bin::gzip","gzip").c_str();
  98             Args[1] = "-d";
  99             Args[2] = 0;
 100             execvp(Args[0],(char **)Args);
 101             exit(100);
 102          }
 103
 104          // Wait for gzip to finish
 105          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 106             return _error->Error("gzip failed, perhaps the disk is full.");
 107
 108          Pkg.Seek(0);
 109       }
 110       pkgTagFile Parser(&Pkg);
 111       if (_error->PendingError() == true)
 112          return false;
 113
 114       // Open the output file
 115       char S[400];
 116       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",Name.c_str(),
 117                (*I).c_str() + CDROM.length(),GetFileName());
 118       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 119       TargetF += URItoFileName(S);
 120       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 121          TargetF = "/dev/null";
 122       FileFd Target(TargetF,FileFd::WriteEmpty);
 123       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 124       if (_error->PendingError() == true)
 125          return false;
 126       if (TargetFl == 0)
 127          return _error->Errno("fdopen","Failed to reopen fd");
 128
 129       // Setup the progress meter
 130       Progress.OverallProgress(CurrentSize,TotalSize,FileSize,
 131                                string("Reading ") + Type() + " Indexes");
 132
 133       // Parse
 134       Progress.SubProgress(Pkg.Size());
 135       pkgTagSection Section;
 136       this->Section = &Section;
 137       string Prefix;
 138       unsigned long Hits = 0;
 139       unsigned long Chop = 0;
 140       while (Parser.Step(Section) == true)
 141       {
 142          Progress.Progress(Parser.Offset());
 143          string File;
 144          unsigned long Size;
 145          if (GetFile(File,Size) == false)
 146          {
 147             fclose(TargetFl);
 148             return false;
 149          }
 150
 151          if (Chop != 0)
 152             File = OrigPath + ChopDirs(File,Chop);
 153
 154          // See if the file exists
 155          bool Mangled = false;
 156          if (NoStat == false || Hits < 10)
 157          {
 158             // Attempt to fix broken structure
 159             if (Hits == 0)
 160             {
 161                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 162                    ReconstructChop(Chop,*I,File) == false)
 163                {
 164                   if (Debug == true)
 165                      clog << "Missed: " << File << endl;
 166                   NotFound++;
 167                   continue;
 168                }
 169                if (Chop != 0)
 170                   File = OrigPath + ChopDirs(File,Chop);
 171             }
 172
 173             // Get the size
 174             struct stat Buf;
 175             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 176                 Buf.st_size == 0)
 177             {
 178                // Attempt to fix busted symlink support for one instance
 179                string OrigFile = File;
 180                string::size_type Start = File.find("binary-");
 181                string::size_type End = File.find("/",Start+3);
 182                if (Start != string::npos && End != string::npos)
 183                {
 184                   File.replace(Start,End-Start,"binary-all");
 185                   Mangled = true;
 186                }
 187
 188                if (Mangled == false ||
 189                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 190                {
 191                   if (Debug == true)
 192                      clog << "Missed(2): " << OrigFile << endl;
 193                   NotFound++;
 194                   continue;
 195                }
 196             }
 197
 198             // Size match
 199             if ((unsigned)Buf.st_size != Size)
 200             {
 201                if (Debug == true)
 202                   clog << "Wrong Size: " << File << endl;
 203                WrongSize++;
 204                continue;
 205             }
 206          }
 207
 208          Packages++;
 209          Hits++;
 210
 211          if (RewriteEntry(TargetFl,File) == false)
 212          {
 213             fclose(TargetFl);
 214             return false;
 215          }
 216       }
 217       fclose(TargetFl);
 218
 219       if (Debug == true)
 220          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 221
 222       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 223       {
 224          // Move out of the partial directory
 225          Target.Close();
 226          string FinalF = _config->FindDir("Dir::State::lists");
 227          FinalF += URItoFileName(S);
 228          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 229             return _error->Errno("rename","Failed to rename");
 230
 231          // Copy the release file
 232          snprintf(S,sizeof(S),"cdrom:[%s]/%sRelease",Name.c_str(),
 233                   (*I).c_str() + CDROM.length());
 234          string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 235          TargetF += URItoFileName(S);
 236          if (FileExists(*I + "Release") == true)
 237          {
 238             FileFd Target(TargetF,FileFd::WriteEmpty);
 239             FileFd Rel(*I + "Release",FileFd::ReadOnly);
 240             if (_error->PendingError() == true)
 241                return false;
 242
 243             if (CopyFile(Rel,Target) == false)
 244                return false;
 245          }
 246          else
 247          {
 248             // Empty release file
 249             FileFd Target(TargetF,FileFd::WriteEmpty);
 250          }
 251
 252          // Rename the release file
 253          FinalF = _config->FindDir("Dir::State::lists");
 254          FinalF += URItoFileName(S);
 255          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 256             return _error->Errno("rename","Failed to rename");
 257       }
 258
 259       /* Mangle the source to be in the proper notation with
 260          prefix dist [component] */
 261       *I = string(*I,Prefix.length());
 262       ConvertToSourceList(CDROM,*I);
 263       *I = Prefix + ' ' + *I;
 264
 265       CurrentSize += FileSize;
 266    }
 267    Progress.Done();
 268
 269    // Some stats
 270    cout << "Wrote " << Packages << " records" ;
 271    if (NotFound != 0)
 272       cout << " with " << NotFound << " missing files";
 273    if (NotFound != 0 && WrongSize != 0)
 274       cout << " and";
 275    if (WrongSize != 0)
 276       cout << " with " << WrongSize << " mismatched files";
 277    cout << '.' << endl;
 278
 279    if (Packages == 0)
 280       return _error->Warning("No valid records were found.");
 281
 282    if (NotFound + WrongSize > 10)
 283       cout << "Alot of entries were discarded, something may be wrong." << endl;
 284
 285    return true;
 286 }
 287                                                                         /*}}}*/
 288 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 289 // ---------------------------------------------------------------------
 290 /* */
 291 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 292 {
 293    string::size_type I = 0;
 294    do
 295    {
 296       I = Path.find('/',I+1);
 297       Depth--;
 298    }
 299    while (I != string::npos && Depth != 0);
 300
 301    if (I == string::npos)
 302       return string();
 303
 304    return string(Path,I+1);
 305 }
 306                                                                         /*}}}*/
 307 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 308 // ---------------------------------------------------------------------
 309 /* This prepends dir components from the path to the package files to
 310    the path to the deb until it is found */
 311 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 312                                   string File)
 313 {
 314    bool Debug = _config->FindB("Debug::aptcdrom",false);
 315    unsigned int Depth = 1;
 316    string MyPrefix = Prefix;
 317    while (1)
 318    {
 319       struct stat Buf;
 320       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 321       {
 322          if (Debug == true)
 323             cout << "Failed, " << CD + MyPrefix + File << endl;
 324          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 325             continue;
 326
 327          return false;
 328       }
 329       else
 330       {
 331          Prefix = MyPrefix;
 332          return true;
 333       }
 334    }
 335    return false;
 336 }
 337                                                                         /*}}}*/
 338 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 339 // ---------------------------------------------------------------------
 340 /* This removes path components from the filename and prepends the location
 341    of the package files until a file is found */
 342 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 343 {
 344    // Attempt to reconstruct the filename
 345    unsigned long Depth = 0;
 346    while (1)
 347    {
 348       struct stat Buf;
 349       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 350       {
 351          File = ChopDirs(File,1);
 352          Depth++;
 353          if (File.empty() == false)
 354             continue;
 355          return false;
 356       }
 357       else
 358       {
 359          Chop = Depth;
 360          return true;
 361       }
 362    }
 363    return false;
 364 }
 365                                                                         /*}}}*/
 366 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 367 // ---------------------------------------------------------------------
 368 /* We look for things in dists/ notation and convert them to
 369    <dist> <component> form otherwise it is left alone. This also strips
 370    the CD path.
 371
 372    This implements a regex sort of like:
 373     (.*)/dists/([^/]*)/(.*)/binary-*
 374      ^          ^      ^- Component
 375      |          |-------- Distribution
 376      |------------------- Path
 377
 378    It was deciced to use only a single word for dist (rather than say
 379    unstable/non-us) to increase the chance that each CD gets a single
 380    line in sources.list.
 381  */
 382 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 383 {
 384    char S[300];
 385    snprintf(S,sizeof(S),"binary-%s",_config->Find("Apt::Architecture").c_str());
 386
 387    // Strip the cdrom base path
 388    Path = string(Path,CD.length());
 389    if (Path.empty() == true)
 390       Path = "/";
 391
 392    // Too short to be a dists/ type
 393    if (Path.length() < strlen("dists/"))
 394       return;
 395
 396    // Not a dists type.
 397    if (stringcmp(Path.begin(),Path.begin()+strlen("dists/"),"dists/") != 0)
 398       return;
 399
 400    // Isolate the dist
 401    string::size_type Slash = strlen("dists/");
 402    string::size_type Slash2 = Path.find('/',Slash + 1);
 403    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 404       return;
 405    string Dist = string(Path,Slash,Slash2 - Slash);
 406
 407    // Isolate the component
 408    Slash = Slash2;
 409    for (unsigned I = 0; I != 10; I++)
 410    {
 411       Slash = Path.find('/',Slash+1);
 412       if (Slash == string::npos || Slash + 2 >= Path.length())
 413          return;
 414       string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 415
 416       // Verify the trailing binary- bit
 417       string::size_type BinSlash = Path.find('/',Slash + 1);
 418       if (Slash == string::npos)
 419          return;
 420       string Binary = string(Path,Slash+1,BinSlash - Slash-1);
 421
 422       if (Binary != S && Binary != "source")
 423          continue;
 424
 425       Path = Dist + ' ' + Comp;
 426       return;
 427    }
 428 }
 429                                                                         /*}}}*/
 430 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 431 // ---------------------------------------------------------------------
 432 /* */
 433 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 434 {
 435    string::size_type I = 0;
 436    do
 437    {
 438       I = Path.find('/',I+1);
 439       Depth--;
 440    }
 441    while (I != string::npos && Depth != 0);
 442
 443    if (I == string::npos)
 444       return false;
 445
 446    To = string(Path,0,I+1);
 447    return true;
 448 }
 449                                                                         /*}}}*/
 450 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 451 // ---------------------------------------------------------------------
 452 /* */
 453 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 454 {
 455    File = Section->FindS("Filename");
 456    Size = Section->FindI("Size");
 457    if (File.empty() || Size == 0)
 458       return _error->Error("Cannot find filename or size tag");
 459    return true;
 460 }
 461                                                                         /*}}}*/
 462 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 463 // ---------------------------------------------------------------------
 464 /* */
 465 bool PackageCopy::RewriteEntry(FILE *Target,string File)
 466 {
 467    TFRewriteData Changes[] = {{"Filename",File.c_str()},
 468                               {}};
 469
 470    if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
 471       return false;
 472    fputc('\n',Target);
 473    return true;
 474 }
 475                                                                         /*}}}*/
 476 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 477 // ---------------------------------------------------------------------
 478 /* */
 479 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 480 {
 481    string Files = Section->FindS("Files");
 482    if (Files.empty() == true)
 483       return false;
 484
 485    // Stash the / terminated directory prefix
 486    string Base = Section->FindS("Directory");
 487    if (Base.empty() == false && Base[Base.length()-1] != '/')
 488       Base += '/';
 489
 490    // Read the first file triplet
 491    const char *C = Files.c_str();
 492    string sSize;
 493    string MD5Hash;
 494
 495    // Parse each of the elements
 496    if (ParseQuoteWord(C,MD5Hash) == false ||
 497        ParseQuoteWord(C,sSize) == false ||
 498        ParseQuoteWord(C,File) == false)
 499       return _error->Error("Error parsing file record");
 500
 501    // Parse the size and append the directory
 502    Size = atoi(sSize.c_str());
 503    File = Base + File;
 504    return true;
 505 }
 506                                                                         /*}}}*/
 507 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 508 // ---------------------------------------------------------------------
 509 /* */
 510 bool SourceCopy::RewriteEntry(FILE *Target,string File)
 511 {
 512    string Dir(File,0,File.rfind('/'));
 513    TFRewriteData Changes[] = {{"Directory",Dir.c_str()},
 514                               {}};
 515
 516    if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
 517       return false;
 518    fputc('\n',Target);
 519    return true;
 520 }
 521                                                                         /*}}}*/