apt-pkg/acquire-worker.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-worker.cc,v 1.34 2001/05/22 04:42:54 jgg Exp $
   4 /* ######################################################################
   5
   6    Acquire Worker
   7
   8    The worker process can startup either as a Configuration prober
   9    or as a queue runner. As a configuration prober it only reads the
  10    configuration message and
  11
  12    ##################################################################### */
  13                                                                         /*}}}*/
  14 // Include Files                                                        /*{{{*/
  15 #include <config.h>
  16
  17 #include <apt-pkg/acquire.h>
  18 #include <apt-pkg/acquire-worker.h>
  19 #include <apt-pkg/acquire-item.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/fileutl.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/hashes.h>
  25
  26 #include <string>
  27 #include <vector>
  28 #include <iostream>
  29 #include <sstream>
  30
  31 #include <sys/stat.h>
  32 #include <stdlib.h>
  33 #include <unistd.h>
  34 #include <signal.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37
  38 #include <apti18n.h>
  39                                                                         /*}}}*/
  40
  41 using namespace std;
  42
  43 // Worker::Worker - Constructor for Queue startup                       /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* */
  46 pkgAcquire::Worker::Worker(Queue *Q,MethodConfig *Cnf,
  47                            pkgAcquireStatus *Log) : Log(Log)
  48 {
  49    OwnerQ = Q;
  50    Config = Cnf;
  51    Access = Cnf->Access;
  52    CurrentItem = 0;
  53    TotalSize = 0;
  54    CurrentSize = 0;
  55
  56    Construct();
  57 }
  58                                                                         /*}}}*/
  59 // Worker::Worker - Constructor for method config startup               /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 pkgAcquire::Worker::Worker(MethodConfig *Cnf)
  63 {
  64    OwnerQ = 0;
  65    Config = Cnf;
  66    Access = Cnf->Access;
  67    CurrentItem = 0;
  68    TotalSize = 0;
  69    CurrentSize = 0;
  70
  71    Construct();
  72 }
  73                                                                         /*}}}*/
  74 // Worker::Construct - Constructor helper                               /*{{{*/
  75 // ---------------------------------------------------------------------
  76 /* */
  77 void pkgAcquire::Worker::Construct()
  78 {
  79    NextQueue = 0;
  80    NextAcquire = 0;
  81    Process = -1;
  82    InFd = -1;
  83    OutFd = -1;
  84    OutReady = false;
  85    InReady = false;
  86    Debug = _config->FindB("Debug::pkgAcquire::Worker",false);
  87 }
  88                                                                         /*}}}*/
  89 // Worker::~Worker - Destructor                                         /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* */
  92 pkgAcquire::Worker::~Worker()
  93 {
  94    close(InFd);
  95    close(OutFd);
  96
  97    if (Process > 0)
  98    {
  99       /* Closing of stdin is the signal to exit and die when the process
 100          indicates it needs cleanup */
 101       if (Config->NeedsCleanup == false)
 102          kill(Process,SIGINT);
 103       ExecWait(Process,Access.c_str(),true);
 104    }
 105 }
 106                                                                         /*}}}*/
 107 // Worker::Start - Start the worker process                             /*{{{*/
 108 // ---------------------------------------------------------------------
 109 /* This forks the method and inits the communication channel */
 110 bool pkgAcquire::Worker::Start()
 111 {
 112    // Get the method path
 113    string Method = _config->FindDir("Dir::Bin::Methods") + Access;
 114    if (FileExists(Method) == false)
 115    {
 116       _error->Error(_("The method driver %s could not be found."),Method.c_str());
 117       if (Access == "https")
 118          _error->Notice(_("Is the package %s installed?"), "apt-transport-https");
 119       return false;
 120    }
 121
 122    if (Debug == true)
 123       clog << "Starting method '" << Method << '\'' << endl;
 124
 125    // Create the pipes
 126    int Pipes[4] = {-1,-1,-1,-1};
 127    if (pipe(Pipes) != 0 || pipe(Pipes+2) != 0)
 128    {
 129       _error->Errno("pipe","Failed to create IPC pipe to subprocess");
 130       for (int I = 0; I != 4; I++)
 131          close(Pipes[I]);
 132       return false;
 133    }
 134    for (int I = 0; I != 4; I++)
 135       SetCloseExec(Pipes[I],true);
 136
 137    // Fork off the process
 138    Process = ExecFork();
 139    if (Process == 0)
 140    {
 141       // Setup the FDs
 142       dup2(Pipes[1],STDOUT_FILENO);
 143       dup2(Pipes[2],STDIN_FILENO);
 144       SetCloseExec(STDOUT_FILENO,false);
 145       SetCloseExec(STDIN_FILENO,false);
 146       SetCloseExec(STDERR_FILENO,false);
 147
 148       const char *Args[2];
 149       Args[0] = Method.c_str();
 150       Args[1] = 0;
 151       execv(Args[0],(char **)Args);
 152       cerr << "Failed to exec method " << Args[0] << endl;
 153       _exit(100);
 154    }
 155
 156    // Fix up our FDs
 157    InFd = Pipes[0];
 158    OutFd = Pipes[3];
 159    SetNonBlock(Pipes[0],true);
 160    SetNonBlock(Pipes[3],true);
 161    close(Pipes[1]);
 162    close(Pipes[2]);
 163    OutReady = false;
 164    InReady = true;
 165
 166    // Read the configuration data
 167    if (WaitFd(InFd) == false ||
 168        ReadMessages() == false)
 169       return _error->Error(_("Method %s did not start correctly"),Method.c_str());
 170
 171    RunMessages();
 172    if (OwnerQ != 0)
 173       SendConfiguration();
 174
 175    return true;
 176 }
 177                                                                         /*}}}*/
 178 // Worker::ReadMessages - Read all pending messages into the list       /*{{{*/
 179 // ---------------------------------------------------------------------
 180 /* */
 181 bool pkgAcquire::Worker::ReadMessages()
 182 {
 183    if (::ReadMessages(InFd,MessageQueue) == false)
 184       return MethodFailure();
 185    return true;
 186 }
 187                                                                         /*}}}*/
 188 // Worker::RunMessage - Empty the message queue                         /*{{{*/
 189 // ---------------------------------------------------------------------
 190 /* This takes the messages from the message queue and runs them through
 191    the parsers in order. */
 192 bool pkgAcquire::Worker::RunMessages()
 193 {
 194    while (MessageQueue.empty() == false)
 195    {
 196       string Message = MessageQueue.front();
 197       MessageQueue.erase(MessageQueue.begin());
 198
 199       if (Debug == true)
 200          clog << " <- " << Access << ':' << QuoteString(Message,"\n") << endl;
 201
 202       // Fetch the message number
 203       char *End;
 204       int Number = strtol(Message.c_str(),&End,10);
 205       if (End == Message.c_str())
 206          return _error->Error("Invalid message from method %s: %s",Access.c_str(),Message.c_str());
 207
 208       string URI = LookupTag(Message,"URI");
 209       pkgAcquire::Queue::QItem *Itm = 0;
 210       if (URI.empty() == false)
 211          Itm = OwnerQ->FindItem(URI,this);
 212
 213       // update used mirror
 214       string UsedMirror = LookupTag(Message,"UsedMirror", "");
 215       if (!UsedMirror.empty() &&
 216           Itm &&
 217           Itm->Description.find(" ") != string::npos)
 218       {
 219          Itm->Description.replace(0, Itm->Description.find(" "), UsedMirror);
 220          // FIXME: will we need this as well?
 221          //Itm->ShortDesc = UsedMirror;
 222       }
 223
 224       // Determine the message number and dispatch
 225       switch (Number)
 226       {
 227          // 100 Capabilities
 228          case 100:
 229          if (Capabilities(Message) == false)
 230             return _error->Error("Unable to process Capabilities message from %s",Access.c_str());
 231          break;
 232
 233          // 101 Log
 234          case 101:
 235          if (Debug == true)
 236             clog << " <- (log) " << LookupTag(Message,"Message") << endl;
 237          break;
 238
 239          // 102 Status
 240          case 102:
 241          Status = LookupTag(Message,"Message");
 242          break;
 243
 244          // 103 Redirect
 245          case 103:
 246          {
 247             if (Itm == 0)
 248             {
 249                _error->Error("Method gave invalid 103 Redirect message");
 250                break;
 251             }
 252
 253             string NewURI = LookupTag(Message,"New-URI",URI.c_str());
 254             Itm->URI = NewURI;
 255
 256             ItemDone();
 257
 258             pkgAcquire::Item *Owner = Itm->Owner;
 259             pkgAcquire::ItemDesc Desc = *Itm;
 260
 261             // Change the status so that it can be dequeued
 262             Owner->Status = pkgAcquire::Item::StatIdle;
 263             // Mark the item as done (taking care of all queues)
 264             // and then put it in the main queue again
 265             OwnerQ->ItemDone(Itm);
 266             OwnerQ->Owner->Enqueue(Desc);
 267
 268             if (Log != 0)
 269                Log->Done(Desc);
 270             break;
 271          }
 272
 273          // 200 URI Start
 274          case 200:
 275          {
 276             if (Itm == 0)
 277             {
 278                _error->Error("Method gave invalid 200 URI Start message");
 279                break;
 280             }
 281
 282             CurrentItem = Itm;
 283             CurrentSize = 0;
 284             TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
 285             ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10);
 286             Itm->Owner->Start(Message,strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10));
 287
 288             // Display update before completion
 289             if (Log != 0 && Log->MorePulses == true)
 290                Log->Pulse(Itm->Owner->GetOwner());
 291
 292             if (Log != 0)
 293                Log->Fetch(*Itm);
 294
 295             break;
 296          }
 297
 298          // 201 URI Done
 299          case 201:
 300          {
 301             if (Itm == 0)
 302             {
 303                _error->Error("Method gave invalid 201 URI Done message");
 304                break;
 305             }
 306
 307             pkgAcquire::Item *Owner = Itm->Owner;
 308             pkgAcquire::ItemDesc Desc = *Itm;
 309
 310             // Display update before completion
 311             if (Log != 0 && Log->MorePulses == true)
 312                Log->Pulse(Owner->GetOwner());
 313
 314             OwnerQ->ItemDone(Itm);
 315             unsigned long long const ServerSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
 316             bool isHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) ||
 317                          StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false);
 318             // Using the https method the server might return 200, but the
 319             // If-Modified-Since condition is not satsified, libcurl will
 320             // discard the download. In this case, however, TotalSize will be
 321             // set to the actual size of the file, while ServerSize will be set
 322             // to 0. Therefore, if the item is marked as a hit and the
 323             // downloaded size (ServerSize) is 0, we ignore TotalSize.
 324             if (TotalSize != 0 && (!isHit || ServerSize != 0) && ServerSize != TotalSize)
 325                _error->Warning("Size of file %s is not what the server reported %s %llu",
 326                                Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize);
 327
 328             // see if there is a hash to verify
 329             HashStringList RecivedHashes;
 330             HashStringList expectedHashes = Owner->HashSums();
 331             for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs)
 332             {
 333                std::string const tagname = hs->HashType() + "-Hash";
 334                std::string const hashsum = LookupTag(Message, tagname.c_str());
 335                if (hashsum.empty() == false)
 336                   RecivedHashes.push_back(HashString(hs->HashType(), hashsum));
 337             }
 338
 339             if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
 340             {
 341                std::clog << "201 URI Done: " << Owner->DescURI() << endl
 342                   << "RecivedHash:" << endl;
 343                for (HashStringList::const_iterator hs = RecivedHashes.begin(); hs != RecivedHashes.end(); ++hs)
 344                   std::clog <<  "\t- " << hs->toStr() << std::endl;
 345                std::clog << "ExpectedHash:" << endl;
 346                for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs)
 347                   std::clog <<  "\t- " << hs->toStr() << std::endl;
 348                std::clog << endl;
 349             }
 350             Owner->Done(Message, ServerSize, RecivedHashes, Config);
 351             ItemDone();
 352
 353             // Log that we are done
 354             if (Log != 0)
 355             {
 356                if (isHit)
 357                {
 358                   /* Hide 'hits' for local only sources - we also manage to
 359                      hide gets */
 360                   if (Config->LocalOnly == false)
 361                      Log->IMSHit(Desc);
 362                }
 363                else
 364                   Log->Done(Desc);
 365             }
 366             break;
 367          }
 368
 369          // 400 URI Failure
 370          case 400:
 371          {
 372             if (Itm == 0)
 373             {
 374                std::string const msg = LookupTag(Message,"Message");
 375                _error->Error("Method gave invalid 400 URI Failure message: %s", msg.c_str());
 376                break;
 377             }
 378
 379             // Display update before completion
 380             if (Log != 0 && Log->MorePulses == true)
 381                Log->Pulse(Itm->Owner->GetOwner());
 382
 383             pkgAcquire::Item *Owner = Itm->Owner;
 384             pkgAcquire::ItemDesc Desc = *Itm;
 385             OwnerQ->ItemDone(Itm);
 386
 387             // set some status
 388             if(LookupTag(Message,"FailReason") == "Timeout" ||
 389                LookupTag(Message,"FailReason") == "TmpResolveFailure" ||
 390                LookupTag(Message,"FailReason") == "ResolveFailure" ||
 391                LookupTag(Message,"FailReason") == "ConnectionRefused")
 392                Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
 393
 394             Owner->Failed(Message,Config);
 395             ItemDone();
 396
 397             if (Log != 0)
 398                Log->Fail(Desc);
 399
 400             break;
 401          }
 402
 403          // 401 General Failure
 404          case 401:
 405          _error->Error("Method %s General failure: %s",Access.c_str(),LookupTag(Message,"Message").c_str());
 406          break;
 407
 408          // 403 Media Change
 409          case 403:
 410          MediaChange(Message);
 411          break;
 412       }
 413    }
 414    return true;
 415 }
 416                                                                         /*}}}*/
 417 // Worker::Capabilities - 100 Capabilities handler                      /*{{{*/
 418 // ---------------------------------------------------------------------
 419 /* This parses the capabilities message and dumps it into the configuration
 420    structure. */
 421 bool pkgAcquire::Worker::Capabilities(string Message)
 422 {
 423    if (Config == 0)
 424       return true;
 425
 426    Config->Version = LookupTag(Message,"Version");
 427    Config->SingleInstance = StringToBool(LookupTag(Message,"Single-Instance"),false);
 428    Config->Pipeline = StringToBool(LookupTag(Message,"Pipeline"),false);
 429    Config->SendConfig = StringToBool(LookupTag(Message,"Send-Config"),false);
 430    Config->LocalOnly = StringToBool(LookupTag(Message,"Local-Only"),false);
 431    Config->NeedsCleanup = StringToBool(LookupTag(Message,"Needs-Cleanup"),false);
 432    Config->Removable = StringToBool(LookupTag(Message,"Removable"),false);
 433
 434    // Some debug text
 435    if (Debug == true)
 436    {
 437       clog << "Configured access method " << Config->Access << endl;
 438       clog << "Version:" << Config->Version <<
 439               " SingleInstance:" << Config->SingleInstance <<
 440               " Pipeline:" << Config->Pipeline <<
 441               " SendConfig:" << Config->SendConfig <<
 442               " LocalOnly: " << Config->LocalOnly <<
 443               " NeedsCleanup: " << Config->NeedsCleanup <<
 444               " Removable: " << Config->Removable << endl;
 445    }
 446
 447    return true;
 448 }
 449                                                                         /*}}}*/
 450 // Worker::MediaChange - Request a media change                         /*{{{*/
 451 // ---------------------------------------------------------------------
 452 /* */
 453 bool pkgAcquire::Worker::MediaChange(string Message)
 454 {
 455    int status_fd = _config->FindI("APT::Status-Fd",-1);
 456    if(status_fd > 0)
 457    {
 458       string Media = LookupTag(Message,"Media");
 459       string Drive = LookupTag(Message,"Drive");
 460       ostringstream msg,status;
 461       ioprintf(msg,_("Please insert the disc labeled: "
 462                      "'%s' "
 463                      "in the drive '%s' and press enter."),
 464                Media.c_str(),Drive.c_str());
 465       status << "media-change: "  // message
 466              << Media  << ":"     // media
 467              << Drive  << ":"     // drive
 468              << msg.str()         // l10n message
 469              << endl;
 470
 471       std::string const dlstatus = status.str();
 472       FileFd::Write(status_fd, dlstatus.c_str(), dlstatus.size());
 473    }
 474
 475    if (Log == 0 || Log->MediaChange(LookupTag(Message,"Media"),
 476                                     LookupTag(Message,"Drive")) == false)
 477    {
 478       char S[300];
 479       snprintf(S,sizeof(S),"603 Media Changed\nFailed: true\n\n");
 480       if (Debug == true)
 481          clog << " -> " << Access << ':' << QuoteString(S,"\n") << endl;
 482       OutQueue += S;
 483       OutReady = true;
 484       return true;
 485    }
 486
 487    char S[300];
 488    snprintf(S,sizeof(S),"603 Media Changed\n\n");
 489    if (Debug == true)
 490       clog << " -> " << Access << ':' << QuoteString(S,"\n") << endl;
 491    OutQueue += S;
 492    OutReady = true;
 493    return true;
 494 }
 495                                                                         /*}}}*/
 496 // Worker::SendConfiguration - Send the config to the method            /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* */
 499 bool pkgAcquire::Worker::SendConfiguration()
 500 {
 501    if (Config->SendConfig == false)
 502       return true;
 503
 504    if (OutFd == -1)
 505       return false;
 506
 507    /* Write out all of the configuration directives by walking the
 508       configuration tree */
 509    std::ostringstream Message;
 510    Message << "601 Configuration\n";
 511    _config->Dump(Message, NULL, "Config-Item: %F=%V\n", false);
 512    Message << '\n';
 513
 514    if (Debug == true)
 515       clog << " -> " << Access << ':' << QuoteString(Message.str(),"\n") << endl;
 516    OutQueue += Message.str();
 517    OutReady = true;
 518
 519    return true;
 520 }
 521                                                                         /*}}}*/
 522 // Worker::QueueItem - Add an item to the outbound queue                /*{{{*/
 523 // ---------------------------------------------------------------------
 524 /* Send a URI Acquire message to the method */
 525 bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
 526 {
 527    if (OutFd == -1)
 528       return false;
 529
 530    string Message = "600 URI Acquire\n";
 531    Message.reserve(300);
 532    Message += "URI: " + Item->URI;
 533    Message += "\nFilename: " + Item->Owner->DestFile;
 534    HashStringList const hsl = Item->Owner->HashSums();
 535    for (HashStringList::const_iterator hs = hsl.begin(); hs != hsl.end(); ++hs)
 536       Message += "\nExpected-" + hs->HashType() + ": " + hs->HashValue();
 537    if(Item->Owner->FileSize > 0)
 538    {
 539       string MaximumSize;
 540       strprintf(MaximumSize, "%llu", Item->Owner->FileSize);
 541       Message += "\nMaximum-Size: " + MaximumSize;
 542    }
 543    Message += Item->Owner->Custom600Headers();
 544    Message += "\n\n";
 545
 546    if (Debug == true)
 547       clog << " -> " << Access << ':' << QuoteString(Message,"\n") << endl;
 548    OutQueue += Message;
 549    OutReady = true;
 550
 551    return true;
 552 }
 553                                                                         /*}}}*/
 554 // Worker::OutFdRead - Out bound FD is ready                            /*{{{*/
 555 // ---------------------------------------------------------------------
 556 /* */
 557 bool pkgAcquire::Worker::OutFdReady()
 558 {
 559    int Res;
 560    do
 561    {
 562       Res = write(OutFd,OutQueue.c_str(),OutQueue.length());
 563    }
 564    while (Res < 0 && errno == EINTR);
 565
 566    if (Res <= 0)
 567       return MethodFailure();
 568
 569    OutQueue.erase(0,Res);
 570    if (OutQueue.empty() == true)
 571       OutReady = false;
 572
 573    return true;
 574 }
 575                                                                         /*}}}*/
 576 // Worker::InFdRead - In bound FD is ready                              /*{{{*/
 577 // ---------------------------------------------------------------------
 578 /* */
 579 bool pkgAcquire::Worker::InFdReady()
 580 {
 581    if (ReadMessages() == false)
 582       return false;
 583    RunMessages();
 584    return true;
 585 }
 586                                                                         /*}}}*/
 587 // Worker::MethodFailure - Called when the method fails                 /*{{{*/
 588 // ---------------------------------------------------------------------
 589 /* This is called when the method is believed to have failed, probably because
 590    read returned -1. */
 591 bool pkgAcquire::Worker::MethodFailure()
 592 {
 593    _error->Error("Method %s has died unexpectedly!",Access.c_str());
 594
 595    // do not reap the child here to show meaningfull error to the user
 596    ExecWait(Process,Access.c_str(),false);
 597    Process = -1;
 598    close(InFd);
 599    close(OutFd);
 600    InFd = -1;
 601    OutFd = -1;
 602    OutReady = false;
 603    InReady = false;
 604    OutQueue = string();
 605    MessageQueue.erase(MessageQueue.begin(),MessageQueue.end());
 606
 607    return false;
 608 }
 609                                                                         /*}}}*/
 610 // Worker::Pulse - Called periodically                                  /*{{{*/
 611 // ---------------------------------------------------------------------
 612 /* */
 613 void pkgAcquire::Worker::Pulse()
 614 {
 615    if (CurrentItem == 0)
 616       return;
 617
 618    struct stat Buf;
 619    if (stat(CurrentItem->Owner->DestFile.c_str(),&Buf) != 0)
 620       return;
 621    CurrentSize = Buf.st_size;
 622
 623    // Hmm? Should not happen...
 624    if (CurrentSize > TotalSize && TotalSize != 0)
 625       TotalSize = CurrentSize;
 626 }
 627                                                                         /*}}}*/
 628 // Worker::ItemDone - Called when the current item is finished          /*{{{*/
 629 // ---------------------------------------------------------------------
 630 /* */
 631 void pkgAcquire::Worker::ItemDone()
 632 {
 633    CurrentItem = 0;
 634    CurrentSize = 0;
 635    TotalSize = 0;
 636    Status = string();
 637 }
 638                                                                         /*}}}*/