apt-pkg/acquire-worker.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-worker.cc,v 1.34 2001/05/22 04:42:54 jgg Exp $
   4 /* ######################################################################
   5
   6    Acquire Worker
   7
   8    The worker process can startup either as a Configuration prober
   9    or as a queue runner. As a configuration prober it only reads the
  10    configuration message and
  11
  12    ##################################################################### */
  13                                                                         /*}}}*/
  14 // Include Files                                                        /*{{{*/
  15 #include <config.h>
  16
  17 #include <apt-pkg/acquire-worker.h>
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/fileutl.h>
  22 #include <apt-pkg/strutl.h>
  23
  24 #include <iostream>
  25 #include <sstream>
  26 #include <fstream>
  27
  28 #include <sys/stat.h>
  29 #include <unistd.h>
  30 #include <fcntl.h>
  31 #include <signal.h>
  32 #include <stdio.h>
  33 #include <errno.h>
  34
  35 #include <apti18n.h>
  36                                                                         /*}}}*/
  37
  38 using namespace std;
  39
  40 // Worker::Worker - Constructor for Queue startup                       /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 pkgAcquire::Worker::Worker(Queue *Q,MethodConfig *Cnf,
  44                            pkgAcquireStatus *Log) : Log(Log)
  45 {
  46    OwnerQ = Q;
  47    Config = Cnf;
  48    Access = Cnf->Access;
  49    CurrentItem = 0;
  50    TotalSize = 0;
  51    CurrentSize = 0;
  52
  53    Construct();
  54 }
  55                                                                         /*}}}*/
  56 // Worker::Worker - Constructor for method config startup               /*{{{*/
  57 // ---------------------------------------------------------------------
  58 /* */
  59 pkgAcquire::Worker::Worker(MethodConfig *Cnf)
  60 {
  61    OwnerQ = 0;
  62    Config = Cnf;
  63    Access = Cnf->Access;
  64    CurrentItem = 0;
  65    TotalSize = 0;
  66    CurrentSize = 0;
  67
  68    Construct();
  69 }
  70                                                                         /*}}}*/
  71 // Worker::Construct - Constructor helper                               /*{{{*/
  72 // ---------------------------------------------------------------------
  73 /* */
  74 void pkgAcquire::Worker::Construct()
  75 {
  76    NextQueue = 0;
  77    NextAcquire = 0;
  78    Process = -1;
  79    InFd = -1;
  80    OutFd = -1;
  81    OutReady = false;
  82    InReady = false;
  83    Debug = _config->FindB("Debug::pkgAcquire::Worker",false);
  84 }
  85                                                                         /*}}}*/
  86 // Worker::~Worker - Destructor                                         /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* */
  89 pkgAcquire::Worker::~Worker()
  90 {
  91    close(InFd);
  92    close(OutFd);
  93
  94    if (Process > 0)
  95    {
  96       /* Closing of stdin is the signal to exit and die when the process
  97          indicates it needs cleanup */
  98       if (Config->NeedsCleanup == false)
  99          kill(Process,SIGINT);
 100       ExecWait(Process,Access.c_str(),true);
 101    }
 102 }
 103                                                                         /*}}}*/
 104 // Worker::Start - Start the worker process                             /*{{{*/
 105 // ---------------------------------------------------------------------
 106 /* This forks the method and inits the communication channel */
 107 bool pkgAcquire::Worker::Start()
 108 {
 109    // Get the method path
 110    string Method = _config->FindDir("Dir::Bin::Methods") + Access;
 111    if (FileExists(Method) == false)
 112       return _error->Error(_("The method driver %s could not be found."),Method.c_str());
 113
 114    if (Debug == true)
 115       clog << "Starting method '" << Method << '\'' << endl;
 116
 117    // Create the pipes
 118    int Pipes[4] = {-1,-1,-1,-1};
 119    if (pipe(Pipes) != 0 || pipe(Pipes+2) != 0)
 120    {
 121       _error->Errno("pipe","Failed to create IPC pipe to subprocess");
 122       for (int I = 0; I != 4; I++)
 123          close(Pipes[I]);
 124       return false;
 125    }
 126    for (int I = 0; I != 4; I++)
 127       SetCloseExec(Pipes[I],true);
 128
 129    // Fork off the process
 130    Process = ExecFork();
 131    if (Process == 0)
 132    {
 133       // Setup the FDs
 134       dup2(Pipes[1],STDOUT_FILENO);
 135       dup2(Pipes[2],STDIN_FILENO);
 136       SetCloseExec(STDOUT_FILENO,false);
 137       SetCloseExec(STDIN_FILENO,false);
 138       SetCloseExec(STDERR_FILENO,false);
 139
 140       const char *Args[2];
 141       Args[0] = Method.c_str();
 142       Args[1] = 0;
 143       execv(Args[0],(char **)Args);
 144       cerr << "Failed to exec method " << Args[0] << endl;
 145       _exit(100);
 146    }
 147
 148    // Fix up our FDs
 149    InFd = Pipes[0];
 150    OutFd = Pipes[3];
 151    SetNonBlock(Pipes[0],true);
 152    SetNonBlock(Pipes[3],true);
 153    close(Pipes[1]);
 154    close(Pipes[2]);
 155    OutReady = false;
 156    InReady = true;
 157
 158    // Read the configuration data
 159    if (WaitFd(InFd) == false ||
 160        ReadMessages() == false)
 161       return _error->Error(_("Method %s did not start correctly"),Method.c_str());
 162
 163    RunMessages();
 164    if (OwnerQ != 0)
 165       SendConfiguration();
 166
 167    return true;
 168 }
 169                                                                         /*}}}*/
 170 // Worker::ReadMessages - Read all pending messages into the list       /*{{{*/
 171 // ---------------------------------------------------------------------
 172 /* */
 173 bool pkgAcquire::Worker::ReadMessages()
 174 {
 175    if (::ReadMessages(InFd,MessageQueue) == false)
 176       return MethodFailure();
 177    return true;
 178 }
 179                                                                         /*}}}*/
 180 // Worker::RunMessage - Empty the message queue                         /*{{{*/
 181 // ---------------------------------------------------------------------
 182 /* This takes the messages from the message queue and runs them through
 183    the parsers in order. */
 184 bool pkgAcquire::Worker::RunMessages()
 185 {
 186    while (MessageQueue.empty() == false)
 187    {
 188       string Message = MessageQueue.front();
 189       MessageQueue.erase(MessageQueue.begin());
 190
 191       if (Debug == true)
 192          clog << " <- " << Access << ':' << QuoteString(Message,"\n") << endl;
 193
 194       // Fetch the message number
 195       char *End;
 196       int Number = strtol(Message.c_str(),&End,10);
 197       if (End == Message.c_str())
 198          return _error->Error("Invalid message from method %s: %s",Access.c_str(),Message.c_str());
 199
 200       string URI = LookupTag(Message,"URI");
 201       pkgAcquire::Queue::QItem *Itm = 0;
 202       if (URI.empty() == false)
 203          Itm = OwnerQ->FindItem(URI,this);
 204
 205       // update used mirror
 206       string UsedMirror = LookupTag(Message,"UsedMirror", "");
 207       if (!UsedMirror.empty() &&
 208           Itm &&
 209           Itm->Description.find(" ") != string::npos)
 210       {
 211          Itm->Description.replace(0, Itm->Description.find(" "), UsedMirror);
 212          // FIXME: will we need this as well?
 213          //Itm->ShortDesc = UsedMirror;
 214       }
 215
 216       // Determine the message number and dispatch
 217       switch (Number)
 218       {
 219          // 100 Capabilities
 220          case 100:
 221          if (Capabilities(Message) == false)
 222             return _error->Error("Unable to process Capabilities message from %s",Access.c_str());
 223          break;
 224
 225          // 101 Log
 226          case 101:
 227          if (Debug == true)
 228             clog << " <- (log) " << LookupTag(Message,"Message") << endl;
 229          break;
 230
 231          // 102 Status
 232          case 102:
 233          Status = LookupTag(Message,"Message");
 234          break;
 235
 236          // 103 Redirect
 237          case 103:
 238          {
 239             if (Itm == 0)
 240             {
 241                _error->Error("Method gave invalid 103 Redirect message");
 242                break;
 243             }
 244
 245             string NewURI = LookupTag(Message,"New-URI",URI.c_str());
 246             Itm->URI = NewURI;
 247
 248             ItemDone();
 249
 250             pkgAcquire::Item *Owner = Itm->Owner;
 251             pkgAcquire::ItemDesc Desc = *Itm;
 252
 253             // Change the status so that it can be dequeued
 254             Owner->Status = pkgAcquire::Item::StatIdle;
 255             // Mark the item as done (taking care of all queues)
 256             // and then put it in the main queue again
 257             OwnerQ->ItemDone(Itm);
 258             OwnerQ->Owner->Enqueue(Desc);
 259
 260             if (Log != 0)
 261                Log->Done(Desc);
 262             break;
 263          }
 264
 265          // 200 URI Start
 266          case 200:
 267          {
 268             if (Itm == 0)
 269             {
 270                _error->Error("Method gave invalid 200 URI Start message");
 271                break;
 272             }
 273
 274             CurrentItem = Itm;
 275             CurrentSize = 0;
 276             TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
 277             ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10);
 278             Itm->Owner->Start(Message,strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10));
 279
 280             // Display update before completion
 281             if (Log != 0 && Log->MorePulses == true)
 282                Log->Pulse(Itm->Owner->GetOwner());
 283
 284             if (Log != 0)
 285                Log->Fetch(*Itm);
 286
 287             break;
 288          }
 289
 290          // 201 URI Done
 291          case 201:
 292          {
 293             if (Itm == 0)
 294             {
 295                _error->Error("Method gave invalid 201 URI Done message");
 296                break;
 297             }
 298
 299             pkgAcquire::Item *Owner = Itm->Owner;
 300             pkgAcquire::ItemDesc Desc = *Itm;
 301
 302             // Display update before completion
 303             if (Log != 0 && Log->MorePulses == true)
 304                Log->Pulse(Owner->GetOwner());
 305
 306             OwnerQ->ItemDone(Itm);
 307             unsigned long long const ServerSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
 308             bool isHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) ||
 309                          StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false);
 310             // Using the https method the server might return 200, but the
 311             // If-Modified-Since condition is not satsified, libcurl will
 312             // discard the download. In this case, however, TotalSize will be
 313             // set to the actual size of the file, while ServerSize will be set
 314             // to 0. Therefore, if the item is marked as a hit and the
 315             // downloaded size (ServerSize) is 0, we ignore TotalSize.
 316             if (TotalSize != 0 && (!isHit || ServerSize != 0) && ServerSize != TotalSize)
 317                _error->Warning("Size of file %s is not what the server reported %s %llu",
 318                                Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize);
 319
 320             // see if there is a hash to verify
 321             string RecivedHash;
 322             HashString expectedHash(Owner->HashSum());
 323             if(!expectedHash.empty())
 324             {
 325                string hashTag = expectedHash.HashType()+"-Hash";
 326                string hashSum = LookupTag(Message, hashTag.c_str());
 327                if(!hashSum.empty())
 328                   RecivedHash = expectedHash.HashType() + ":" + hashSum;
 329                if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
 330                {
 331                   clog << "201 URI Done: " << Owner->DescURI() << endl
 332                        << "RecivedHash: " << RecivedHash << endl
 333                        << "ExpectedHash: " << expectedHash.toStr()
 334                        << endl << endl;
 335                }
 336             }
 337             Owner->Done(Message, ServerSize, RecivedHash.c_str(), Config);
 338             ItemDone();
 339
 340             // Log that we are done
 341             if (Log != 0)
 342             {
 343                if (isHit)
 344                {
 345                   /* Hide 'hits' for local only sources - we also manage to
 346                      hide gets */
 347                   if (Config->LocalOnly == false)
 348                      Log->IMSHit(Desc);
 349                }
 350                else
 351                   Log->Done(Desc);
 352             }
 353             break;
 354          }
 355
 356          // 400 URI Failure
 357          case 400:
 358          {
 359             if (Itm == 0)
 360             {
 361                _error->Error("Method gave invalid 400 URI Failure message");
 362                break;
 363             }
 364
 365             // Display update before completion
 366             if (Log != 0 && Log->MorePulses == true)
 367                Log->Pulse(Itm->Owner->GetOwner());
 368
 369             pkgAcquire::Item *Owner = Itm->Owner;
 370             pkgAcquire::ItemDesc Desc = *Itm;
 371             OwnerQ->ItemDone(Itm);
 372
 373             // set some status
 374             if(LookupTag(Message,"FailReason") == "Timeout" ||
 375                LookupTag(Message,"FailReason") == "TmpResolveFailure" ||
 376                LookupTag(Message,"FailReason") == "ResolveFailure" ||
 377                LookupTag(Message,"FailReason") == "ConnectionRefused")
 378                Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
 379
 380             Owner->Failed(Message,Config);
 381             ItemDone();
 382
 383             if (Log != 0)
 384                Log->Fail(Desc);
 385
 386             break;
 387          }
 388
 389          // 401 General Failure
 390          case 401:
 391          _error->Error("Method %s General failure: %s",Access.c_str(),LookupTag(Message,"Message").c_str());
 392          break;
 393
 394          // 403 Media Change
 395          case 403:
 396          MediaChange(Message);
 397          break;
 398       }
 399    }
 400    return true;
 401 }
 402                                                                         /*}}}*/
 403 // Worker::Capabilities - 100 Capabilities handler                      /*{{{*/
 404 // ---------------------------------------------------------------------
 405 /* This parses the capabilities message and dumps it into the configuration
 406    structure. */
 407 bool pkgAcquire::Worker::Capabilities(string Message)
 408 {
 409    if (Config == 0)
 410       return true;
 411
 412    Config->Version = LookupTag(Message,"Version");
 413    Config->SingleInstance = StringToBool(LookupTag(Message,"Single-Instance"),false);
 414    Config->Pipeline = StringToBool(LookupTag(Message,"Pipeline"),false);
 415    Config->SendConfig = StringToBool(LookupTag(Message,"Send-Config"),false);
 416    Config->LocalOnly = StringToBool(LookupTag(Message,"Local-Only"),false);
 417    Config->NeedsCleanup = StringToBool(LookupTag(Message,"Needs-Cleanup"),false);
 418    Config->Removable = StringToBool(LookupTag(Message,"Removable"),false);
 419
 420    // Some debug text
 421    if (Debug == true)
 422    {
 423       clog << "Configured access method " << Config->Access << endl;
 424       clog << "Version:" << Config->Version <<
 425               " SingleInstance:" << Config->SingleInstance <<
 426               " Pipeline:" << Config->Pipeline <<
 427               " SendConfig:" << Config->SendConfig <<
 428               " LocalOnly: " << Config->LocalOnly <<
 429               " NeedsCleanup: " << Config->NeedsCleanup <<
 430               " Removable: " << Config->Removable << endl;
 431    }
 432
 433    return true;
 434 }
 435                                                                         /*}}}*/
 436 // Worker::MediaChange - Request a media change                         /*{{{*/
 437 // ---------------------------------------------------------------------
 438 /* */
 439 bool pkgAcquire::Worker::MediaChange(string Message)
 440 {
 441    int status_fd = _config->FindI("APT::Status-Fd",-1);
 442    if(status_fd > 0)
 443    {
 444       string Media = LookupTag(Message,"Media");
 445       string Drive = LookupTag(Message,"Drive");
 446       ostringstream msg,status;
 447       ioprintf(msg,_("Please insert the disc labeled: "
 448                      "'%s' "
 449                      "in the drive '%s' and press enter."),
 450                Media.c_str(),Drive.c_str());
 451       status << "media-change: "  // message
 452              << Media  << ":"     // media
 453              << Drive  << ":"     // drive
 454              << msg.str()         // l10n message
 455              << endl;
 456
 457       std::string const dlstatus = status.str();
 458       FileFd::Write(status_fd, dlstatus.c_str(), dlstatus.size());
 459    }
 460
 461    if (Log == 0 || Log->MediaChange(LookupTag(Message,"Media"),
 462                                     LookupTag(Message,"Drive")) == false)
 463    {
 464       char S[300];
 465       snprintf(S,sizeof(S),"603 Media Changed\nFailed: true\n\n");
 466       if (Debug == true)
 467          clog << " -> " << Access << ':' << QuoteString(S,"\n") << endl;
 468       OutQueue += S;
 469       OutReady = true;
 470       return true;
 471    }
 472
 473    char S[300];
 474    snprintf(S,sizeof(S),"603 Media Changed\n\n");
 475    if (Debug == true)
 476       clog << " -> " << Access << ':' << QuoteString(S,"\n") << endl;
 477    OutQueue += S;
 478    OutReady = true;
 479    return true;
 480 }
 481                                                                         /*}}}*/
 482 // Worker::SendConfiguration - Send the config to the method            /*{{{*/
 483 // ---------------------------------------------------------------------
 484 /* */
 485 bool pkgAcquire::Worker::SendConfiguration()
 486 {
 487    if (Config->SendConfig == false)
 488       return true;
 489
 490    if (OutFd == -1)
 491       return false;
 492
 493    /* Write out all of the configuration directives by walking the
 494       configuration tree */
 495    std::ostringstream Message;
 496    Message << "601 Configuration\n";
 497    _config->Dump(Message, NULL, "Config-Item: %F=%V\n", false);
 498    Message << '\n';
 499
 500    if (Debug == true)
 501       clog << " -> " << Access << ':' << QuoteString(Message.str(),"\n") << endl;
 502    OutQueue += Message.str();
 503    OutReady = true;
 504
 505    return true;
 506 }
 507                                                                         /*}}}*/
 508 // Worker::QueueItem - Add an item to the outbound queue                /*{{{*/
 509 // ---------------------------------------------------------------------
 510 /* Send a URI Acquire message to the method */
 511 bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
 512 {
 513    if (OutFd == -1)
 514       return false;
 515
 516    string Message = "600 URI Acquire\n";
 517    Message.reserve(300);
 518    Message += "URI: " + Item->URI;
 519    Message += "\nFilename: " + Item->Owner->DestFile;
 520    Message += Item->Owner->Custom600Headers();
 521    Message += "\n\n";
 522
 523    if (Debug == true)
 524       clog << " -> " << Access << ':' << QuoteString(Message,"\n") << endl;
 525    OutQueue += Message;
 526    OutReady = true;
 527
 528    return true;
 529 }
 530                                                                         /*}}}*/
 531 // Worker::OutFdRead - Out bound FD is ready                            /*{{{*/
 532 // ---------------------------------------------------------------------
 533 /* */
 534 bool pkgAcquire::Worker::OutFdReady()
 535 {
 536    int Res;
 537    do
 538    {
 539       Res = write(OutFd,OutQueue.c_str(),OutQueue.length());
 540    }
 541    while (Res < 0 && errno == EINTR);
 542
 543    if (Res <= 0)
 544       return MethodFailure();
 545
 546    OutQueue.erase(0,Res);
 547    if (OutQueue.empty() == true)
 548       OutReady = false;
 549
 550    return true;
 551 }
 552                                                                         /*}}}*/
 553 // Worker::InFdRead - In bound FD is ready                              /*{{{*/
 554 // ---------------------------------------------------------------------
 555 /* */
 556 bool pkgAcquire::Worker::InFdReady()
 557 {
 558    if (ReadMessages() == false)
 559       return false;
 560    RunMessages();
 561    return true;
 562 }
 563                                                                         /*}}}*/
 564 // Worker::MethodFailure - Called when the method fails                 /*{{{*/
 565 // ---------------------------------------------------------------------
 566 /* This is called when the method is belived to have failed, probably because
 567    read returned -1. */
 568 bool pkgAcquire::Worker::MethodFailure()
 569 {
 570    _error->Error("Method %s has died unexpectedly!",Access.c_str());
 571
 572    // do not reap the child here to show meaningfull error to the user
 573    ExecWait(Process,Access.c_str(),false);
 574    Process = -1;
 575    close(InFd);
 576    close(OutFd);
 577    InFd = -1;
 578    OutFd = -1;
 579    OutReady = false;
 580    InReady = false;
 581    OutQueue = string();
 582    MessageQueue.erase(MessageQueue.begin(),MessageQueue.end());
 583
 584    return false;
 585 }
 586                                                                         /*}}}*/
 587 // Worker::Pulse - Called periodically                                  /*{{{*/
 588 // ---------------------------------------------------------------------
 589 /* */
 590 void pkgAcquire::Worker::Pulse()
 591 {
 592    if (CurrentItem == 0)
 593       return;
 594
 595    struct stat Buf;
 596    if (stat(CurrentItem->Owner->DestFile.c_str(),&Buf) != 0)
 597       return;
 598    CurrentSize = Buf.st_size;
 599
 600    // Hmm? Should not happen...
 601    if (CurrentSize > TotalSize && TotalSize != 0)
 602       TotalSize = CurrentSize;
 603 }
 604                                                                         /*}}}*/
 605 // Worker::ItemDone - Called when the current item is finished          /*{{{*/
 606 // ---------------------------------------------------------------------
 607 /* */
 608 void pkgAcquire::Worker::ItemDone()
 609 {
 610    CurrentItem = 0;
 611    CurrentSize = 0;
 612    TotalSize = 0;
 613    Status = string();
 614 }
 615                                                                         /*}}}*/