cmdline/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.5 2000/05/10 06:02:26 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21
  22 #include <iostream.h>
  23 #include <unistd.h>
  24 #include <sys/stat.h>
  25 #include <stdio.h>
  26                                                                         /*}}}*/
  27
  28 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  29 // ---------------------------------------------------------------------
  30 /* */
  31 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List)
  32 {
  33    if (List.size() == 0)
  34       return true;
  35
  36    OpTextProgress Progress;
  37
  38    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  39    bool Debug = _config->FindB("Debug::aptcdrom",false);
  40
  41    // Prepare the progress indicator
  42    unsigned long TotalSize = 0;
  43    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  44    {
  45       struct stat Buf;
  46       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  47           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  48          return _error->Errno("stat","Stat failed for %s",
  49                               string(*I + GetFileName()).c_str());
  50       TotalSize += Buf.st_size;
  51    }
  52
  53    unsigned long CurrentSize = 0;
  54    unsigned int NotFound = 0;
  55    unsigned int WrongSize = 0;
  56    unsigned int Packages = 0;
  57    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  58    {
  59       string OrigPath = string(*I,CDROM.length());
  60       unsigned long FileSize = 0;
  61
  62       // Open the package file
  63       FileFd Pkg;
  64       if (FileExists(*I + GetFileName()) == true)
  65       {
  66          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  67          FileSize = Pkg.Size();
  68       }
  69       else
  70       {
  71          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  72          if (_error->PendingError() == true)
  73             return false;
  74          FileSize = From.Size();
  75
  76          // Get a temp file
  77          FILE *tmp = tmpfile();
  78          if (tmp == 0)
  79             return _error->Errno("tmpfile","Unable to create a tmp file");
  80          Pkg.Fd(dup(fileno(tmp)));
  81          fclose(tmp);
  82
  83          // Fork gzip
  84          int Process = fork();
  85          if (Process < 0)
  86             return _error->Errno("fork","Couldn't fork gzip");
  87
  88          // The child
  89          if (Process == 0)
  90          {
  91             dup2(From.Fd(),STDIN_FILENO);
  92             dup2(Pkg.Fd(),STDOUT_FILENO);
  93             SetCloseExec(STDIN_FILENO,false);
  94             SetCloseExec(STDOUT_FILENO,false);
  95
  96             const char *Args[3];
  97             Args[0] = _config->Find("Dir::bin::gzip","gzip").c_str();
  98             Args[1] = "-d";
  99             Args[2] = 0;
 100             execvp(Args[0],(char **)Args);
 101             exit(100);
 102          }
 103
 104          // Wait for gzip to finish
 105          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 106             return _error->Error("gzip failed, perhaps the disk is full.");
 107
 108          Pkg.Seek(0);
 109       }
 110       pkgTagFile Parser(Pkg);
 111       if (_error->PendingError() == true)
 112          return false;
 113
 114       // Open the output file
 115       char S[400];
 116       sprintf(S,"cdrom:[%s]/%s%s",Name.c_str(),(*I).c_str() + CDROM.length(),
 117               GetFileName());
 118       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 119       TargetF += URItoFileName(S);
 120       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 121          TargetF = "/dev/null";
 122       FileFd Target(TargetF,FileFd::WriteEmpty);
 123       if (_error->PendingError() == true)
 124          return false;
 125
 126       // Setup the progress meter
 127       Progress.OverallProgress(CurrentSize,TotalSize,FileSize,
 128                                string("Reading ") + Type() + " Indexes");
 129
 130       // Parse
 131       Progress.SubProgress(Pkg.Size());
 132       pkgTagSection Section;
 133       this->Section = &Section;
 134       string Prefix;
 135       unsigned long Hits = 0;
 136       unsigned long Chop = 0;
 137       while (Parser.Step(Section) == true)
 138       {
 139          Progress.Progress(Parser.Offset());
 140          string File;
 141          unsigned long Size;
 142          if (GetFile(File,Size) == false)
 143             return false;
 144
 145          if (Chop != 0)
 146             File = OrigPath + ChopDirs(File,Chop);
 147
 148          // See if the file exists
 149          bool Mangled = false;
 150          if (NoStat == false || Hits < 10)
 151          {
 152             // Attempt to fix broken structure
 153             if (Hits == 0)
 154             {
 155                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 156                    ReconstructChop(Chop,*I,File) == false)
 157                {
 158                   if (Debug == true)
 159                      clog << "Missed: " << File << endl;
 160                   NotFound++;
 161                   continue;
 162                }
 163                if (Chop != 0)
 164                   File = OrigPath + ChopDirs(File,Chop);
 165             }
 166
 167             // Get the size
 168             struct stat Buf;
 169             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 170                 Buf.st_size == 0)
 171             {
 172                // Attempt to fix busted symlink support for one instance
 173                string OrigFile = File;
 174                string::size_type Start = File.find("binary-");
 175                string::size_type End = File.find("/",Start+3);
 176                if (Start != string::npos && End != string::npos)
 177                {
 178                   File.replace(Start,End-Start,"binary-all");
 179                   Mangled = true;
 180                }
 181
 182                if (Mangled == false ||
 183                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 184                {
 185                   if (Debug == true)
 186                      clog << "Missed(2): " << OrigFile << endl;
 187                   NotFound++;
 188                   continue;
 189                }
 190             }
 191
 192             // Size match
 193             if ((unsigned)Buf.st_size != Size)
 194             {
 195                if (Debug == true)
 196                   clog << "Wrong Size: " << File << endl;
 197                WrongSize++;
 198                continue;
 199             }
 200          }
 201
 202          Packages++;
 203          Hits++;
 204
 205          // Copy it to the target package file
 206          if (Chop != 0 || Mangled == true)
 207          {
 208             if (RewriteEntry(Target,File) == false)
 209                continue;
 210          }
 211          else
 212          {
 213             const char *Start;
 214             const char *Stop;
 215             Section.GetSection(Start,Stop);
 216             if (Target.Write(Start,Stop-Start) == false)
 217                return false;
 218          }
 219       }
 220
 221       if (Debug == true)
 222          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 223
 224       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 225       {
 226          // Move out of the partial directory
 227          Target.Close();
 228          string FinalF = _config->FindDir("Dir::State::lists");
 229          FinalF += URItoFileName(S);
 230          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 231             return _error->Errno("rename","Failed to rename");
 232
 233          // Copy the release file
 234          sprintf(S,"cdrom:[%s]/%sRelease",Name.c_str(),(*I).c_str() + CDROM.length());
 235          string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 236          TargetF += URItoFileName(S);
 237          if (FileExists(*I + "Release") == true)
 238          {
 239             FileFd Target(TargetF,FileFd::WriteEmpty);
 240             FileFd Rel(*I + "Release",FileFd::ReadOnly);
 241             if (_error->PendingError() == true)
 242                return false;
 243
 244             if (CopyFile(Rel,Target) == false)
 245                return false;
 246          }
 247          else
 248          {
 249             // Empty release file
 250             FileFd Target(TargetF,FileFd::WriteEmpty);
 251          }
 252
 253          // Rename the release file
 254          FinalF = _config->FindDir("Dir::State::lists");
 255          FinalF += URItoFileName(S);
 256          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 257             return _error->Errno("rename","Failed to rename");
 258       }
 259
 260       /* Mangle the source to be in the proper notation with
 261          prefix dist [component] */
 262       *I = string(*I,Prefix.length());
 263       ConvertToSourceList(CDROM,*I);
 264       *I = Prefix + ' ' + *I;
 265
 266       CurrentSize += FileSize;
 267    }
 268    Progress.Done();
 269
 270    // Some stats
 271    cout << "Wrote " << Packages << " records" ;
 272    if (NotFound != 0)
 273       cout << " with " << NotFound << " missing files";
 274    if (NotFound != 0 && WrongSize != 0)
 275       cout << " and";
 276    if (WrongSize != 0)
 277       cout << " with " << WrongSize << " mismatched files";
 278    cout << '.' << endl;
 279
 280    if (Packages == 0)
 281       return _error->Warning("No valid records were found.");
 282
 283    if (NotFound + WrongSize > 10)
 284       cout << "Alot of entries were discarded, something may be wrong." << endl;
 285
 286    return true;
 287 }
 288                                                                         /*}}}*/
 289 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 290 // ---------------------------------------------------------------------
 291 /* */
 292 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 293 {
 294    string::size_type I = 0;
 295    do
 296    {
 297       I = Path.find('/',I+1);
 298       Depth--;
 299    }
 300    while (I != string::npos && Depth != 0);
 301
 302    if (I == string::npos)
 303       return string();
 304
 305    return string(Path,I+1);
 306 }
 307                                                                         /*}}}*/
 308 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 309 // ---------------------------------------------------------------------
 310 /* This prepends dir components from the path to the package files to
 311    the path to the deb until it is found */
 312 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 313                                   string File)
 314 {
 315    bool Debug = _config->FindB("Debug::aptcdrom",false);
 316    unsigned int Depth = 1;
 317    string MyPrefix = Prefix;
 318    while (1)
 319    {
 320       struct stat Buf;
 321       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 322       {
 323          if (Debug == true)
 324             cout << "Failed, " << CD + MyPrefix + File << endl;
 325          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 326             continue;
 327
 328          return false;
 329       }
 330       else
 331       {
 332          Prefix = MyPrefix;
 333          return true;
 334       }
 335    }
 336    return false;
 337 }
 338                                                                         /*}}}*/
 339 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 340 // ---------------------------------------------------------------------
 341 /* This removes path components from the filename and prepends the location
 342    of the package files until a file is found */
 343 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 344 {
 345    // Attempt to reconstruct the filename
 346    unsigned long Depth = 0;
 347    while (1)
 348    {
 349       struct stat Buf;
 350       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 351       {
 352          File = ChopDirs(File,1);
 353          Depth++;
 354          if (File.empty() == false)
 355             continue;
 356          return false;
 357       }
 358       else
 359       {
 360          Chop = Depth;
 361          return true;
 362       }
 363    }
 364    return false;
 365 }
 366                                                                         /*}}}*/
 367 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* We look for things in dists/ notation and convert them to
 370    <dist> <component> form otherwise it is left alone. This also strips
 371    the CD path.
 372
 373    This implements a regex sort of like:
 374     (.*)/dists/([^/]*)/(.*)/binary-*
 375      ^          ^      ^- Component
 376      |          |-------- Distribution
 377      |------------------- Path
 378
 379    It was deciced to use only a single word for dist (rather than say
 380    unstable/non-us) to increase the chance that each CD gets a single
 381    line in sources.list.
 382  */
 383 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 384 {
 385    char S[300];
 386    sprintf(S,"binary-%s",_config->Find("Apt::Architecture").c_str());
 387
 388    // Strip the cdrom base path
 389    Path = string(Path,CD.length());
 390    if (Path.empty() == true)
 391       Path = "/";
 392
 393    // Too short to be a dists/ type
 394    if (Path.length() < strlen("dists/"))
 395       return;
 396
 397    // Not a dists type.
 398    if (stringcmp(Path.begin(),Path.begin()+strlen("dists/"),"dists/") != 0)
 399       return;
 400
 401    // Isolate the dist
 402    string::size_type Slash = strlen("dists/");
 403    string::size_type Slash2 = Path.find('/',Slash + 1);
 404    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 405       return;
 406    string Dist = string(Path,Slash,Slash2 - Slash);
 407
 408    // Isolate the component
 409    Slash = Slash2;
 410    for (unsigned I = 0; I != 10; I++)
 411    {
 412       Slash = Path.find('/',Slash+1);
 413       if (Slash == string::npos || Slash + 2 >= Path.length())
 414          return;
 415       string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 416
 417       // Verify the trailing binary- bit
 418       string::size_type BinSlash = Path.find('/',Slash + 1);
 419       if (Slash == string::npos)
 420          return;
 421       string Binary = string(Path,Slash+1,BinSlash - Slash-1);
 422
 423       if (Binary != S && Binary != "source")
 424          continue;
 425
 426       Path = Dist + ' ' + Comp;
 427       return;
 428    }
 429 }
 430                                                                         /*}}}*/
 431 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 432 // ---------------------------------------------------------------------
 433 /* */
 434 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 435 {
 436    string::size_type I = 0;
 437    do
 438    {
 439       I = Path.find('/',I+1);
 440       Depth--;
 441    }
 442    while (I != string::npos && Depth != 0);
 443
 444    if (I == string::npos)
 445       return false;
 446
 447    To = string(Path,0,I+1);
 448    return true;
 449 }
 450                                                                         /*}}}*/
 451 // IndexCopy::CopyWithReplace - Copy a section and replace text         /*{{{*/
 452 // ---------------------------------------------------------------------
 453 /* */
 454 bool IndexCopy::CopyWithReplace(FileFd &Target,const char *Tag,string New)
 455 {
 456    // Mangle the output filename
 457    const char *Start;
 458    const char *Stop;
 459    const char *Filename;
 460    Section->Find(Tag,Filename,Stop);
 461
 462    /* We need to rewrite the filename field so we emit
 463       all fields except the filename file and rewrite that one */
 464    for (unsigned int I = 0; I != Section->Count(); I++)
 465    {
 466       Section->Get(Start,Stop,I);
 467       if (Start <= Filename && Stop > Filename)
 468       {
 469          char S[500];
 470          sprintf(S,"%s: %s\n",Tag,New.c_str());
 471          if (I + 1 == Section->Count())
 472             strcat(S,"\n");
 473          if (Target.Write(S,strlen(S)) == false)
 474             return false;
 475       }
 476       else
 477       {
 478          if (Target.Write(Start,Stop-Start) == false)
 479             return false;
 480          if (Stop[-1] != '\n')
 481             if (Target.Write("\n",1) == false)
 482                return false;
 483       }
 484    }
 485    if (Target.Write("\n",1) == false)
 486       return false;
 487 }
 488                                                                         /*}}}*/
 489 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 490 // ---------------------------------------------------------------------
 491 /* */
 492 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 493 {
 494    File = Section->FindS("Filename");
 495    Size = Section->FindI("Size");
 496    if (File.empty() || Size == 0)
 497       return _error->Error("Cannot find filename or size tag");
 498    return true;
 499 }
 500                                                                         /*}}}*/
 501 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 502 // ---------------------------------------------------------------------
 503 /* */
 504 bool PackageCopy::RewriteEntry(FileFd &Target,string File)
 505 {
 506    return CopyWithReplace(Target,"Filename",File);
 507 }
 508                                                                         /*}}}*/
 509 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 510 // ---------------------------------------------------------------------
 511 /* */
 512 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 513 {
 514    string Files = Section->FindS("Files");
 515    if (Files.empty() == true)
 516       return false;
 517
 518    // Stash the / terminated directory prefix
 519    string Base = Section->FindS("Directory");
 520    if (Base.empty() == false && Base[Base.length()-1] != '/')
 521       Base += '/';
 522
 523    // Iterate over the entire list grabbing each triplet
 524    const char *C = Files.c_str();
 525    string sSize;
 526    string MD5Hash;
 527
 528    // Parse each of the elements
 529    if (ParseQuoteWord(C,MD5Hash) == false ||
 530        ParseQuoteWord(C,sSize) == false ||
 531        ParseQuoteWord(C,File) == false)
 532       return _error->Error("Error parsing file record");
 533
 534    // Parse the size and append the directory
 535    Size = atoi(sSize.c_str());
 536    File = Base + File;
 537    return true;
 538 }
 539                                                                         /*}}}*/
 540 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 541 // ---------------------------------------------------------------------
 542 /* */
 543 bool SourceCopy::RewriteEntry(FileFd &Target,string File)
 544 {
 545    return CopyWithReplace(Target,"Directory",
 546                           string(File,0,File.rfind('/')));
 547 }
 548                                                                         /*}}}*/