Fixed dumpavail bug with dup packageS

[apt.git] / methods / http.cc

diff --git a/methods/http.cc b/methods/http.cc
index be7e8610a9a29c10d9754e439f5c4504f67865c2..b5d26d127754867fefbc5f5926127cb778b5990e 100644 (file)
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/

-// $Id: http.cc,v 1.7 1998/11/23 21:28:43 jgg Exp $
+// $Id: http.cc,v 1.26 1999/02/15 00:26:55 jgg Exp $

 /* ######################################################################
 
    HTTP Aquire Method - This is the HTTP aquire method for APT.
 /* ######################################################################
 
    HTTP Aquire Method - This is the HTTP aquire method for APT.


@@ -38,6 +38,7 @@
 #include <unistd.h>
 #include <signal.h>
 #include <stdio.h>
 #include <unistd.h>
 #include <signal.h>
 #include <stdio.h>

+#include <errno.h>

 
 // Internet stuff
 #include <netinet/in.h>
 
 // Internet stuff
 #include <netinet/in.h>


@@ -51,6 +52,8 @@
 string HttpMethod::FailFile;
 int HttpMethod::FailFd = -1;
 time_t HttpMethod::FailTime = 0;
 string HttpMethod::FailFile;
 int HttpMethod::FailFd = -1;
 time_t HttpMethod::FailTime = 0;

+unsigned long PipelineDepth = 10;
+unsigned long TimeOut = 120;

 
 // CircleBuf::CircleBuf - Circular input buffer                                /*{{{*/
 // ---------------------------------------------------------------------
 
 // CircleBuf::CircleBuf - Circular input buffer                                /*{{{*/
 // ---------------------------------------------------------------------


@@ -245,7 +248,7 @@ void CircleBuf::Stats()
 // ---------------------------------------------------------------------
 /* */
 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 // ---------------------------------------------------------------------
 /* */
 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),

-                        In(64*1024), Out(1*1024),
+                        In(64*1024), Out(4*1024),

                         ServerName(Srv)
 {
    Reset();
                         ServerName(Srv)
 {
    Reset();


@@ -267,7 +270,7 @@ bool ServerState::Open()
    Out.Reset();
 
    // Determine the proxy setting
    Out.Reset();
 
    // Determine the proxy setting

-   if (getenv("http_proxy") != 0)
+   if (getenv("http_proxy") == 0)

    {
       string DefProxy = _config->Find("Acquire::http::Proxy");
       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
    {
       string DefProxy = _config->Find("Acquire::http::Proxy");
       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);


@@ -309,7 +312,7 @@ bool ServerState::Open()
 
       // Lookup the host
       hostent *Addr = gethostbyname(Host.c_str());
 
       // Lookup the host
       hostent *Addr = gethostbyname(Host.c_str());

-      if (Addr == 0)
+      if (Addr == 0 || Addr->h_addr_list[0] == 0)

         return _error->Error("Could not resolve '%s'",Host.c_str());
       LastHost = Host;
       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
         return _error->Error("Could not resolve '%s'",Host.c_str());
       LastHost = Host;
       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);


@@ -326,10 +329,30 @@ bool ServerState::Open()
    server.sin_family = AF_INET;
    server.sin_port = htons(Port);
    server.sin_addr = LastHostA;
    server.sin_family = AF_INET;
    server.sin_port = htons(Port);
    server.sin_addr = LastHostA;

-   if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
+   SetNonBlock(ServerFd,true);
+   if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
+       errno != EINPROGRESS)

       return _error->Errno("socket","Could not create a socket");
 
       return _error->Errno("socket","Could not create a socket");
 

-   SetNonBlock(ServerFd,true);
+   /* This implements a timeout for connect by opening the connection
+      nonblocking */
+   fd_set wfds;
+   FD_ZERO(&wfds);
+   FD_SET(ServerFd,&wfds);
+   struct timeval tv;
+   tv.tv_sec = TimeOut;
+   tv.tv_usec = 0;
+   int Res = 0;
+   if ((Res = select(ServerFd+1,0,&wfds,0,&tv)) < 0)
+      return _error->Errno("select","Select failed");
+   if (Res == 0)
+      return _error->Error("Could not connect, connection timed out");
+   unsigned int Err,Len=sizeof(Err);
+   if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
+      return _error->Errno("getsockopt","Failed");
+   if (Err != 0)
+      return _error->Error("Could not connect.");
+   

    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/


@@ -425,7 +448,7 @@ bool ServerState::RunData()
            while ((Last = Owner->Go(false,this)) == true);
            if (Last == false)
               return false;
            while ((Last = Owner->Go(false,this)) == true);
            if (Last == false)
               return false;

-           return true;
+           return !_error->PendingError();

         }
         
         // Transfer the block
         }
         
         // Transfer the block


@@ -467,12 +490,12 @@ bool ServerState::RunData()
            continue;
         
         In.Limit(-1);
            continue;
         
         In.Limit(-1);

-        return true;
+        return !_error->PendingError();

       }
       while (Owner->Go(true,this) == true);
    }
 
       }
       while (Owner->Go(true,this) == true);
    }
 

-   return Owner->Flush(this);
+   return Owner->Flush(this) && !_error->PendingError();

 }
                                                                        /*}}}*/
 // ServerState::HeaderLine - Process a header line                     /*{{{*/
 }
                                                                        /*}}}*/
 // ServerState::HeaderLine - Process a header line                     /*{{{*/


@@ -482,7 +505,7 @@ bool ServerState::HeaderLine(string Line)
 {
    if (Line.empty() == true)
       return true;
 {
    if (Line.empty() == true)
       return true;

-   
+

    // The http server might be trying to do something evil.
    if (Line.length() >= MAXLEN)
       return _error->Error("Got a single header line over %u chars",MAXLEN);
    // The http server might be trying to do something evil.
    if (Line.length() >= MAXLEN)
       return _error->Error("Got a single header line over %u chars",MAXLEN);


@@ -572,9 +595,9 @@ bool ServerState::HeaderLine(string Line)
 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 {
    URI Uri = Itm->Uri;
 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 {
    URI Uri = Itm->Uri;

-   
+

    // The HTTP server expects a hostname with a trailing :port
    // The HTTP server expects a hostname with a trailing :port

-   char Buf[300];
+   char Buf[1000];

    string ProperHost = Uri.Host;
    if (Uri.Port != 0)
    {
    string ProperHost = Uri.Host;
    if (Uri.Port != 0)
    {


@@ -582,6 +605,10 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       ProperHost += Buf;
    }   
       
       ProperHost += Buf;
    }   
       

+   // Just in case.
+   if (Itm->Uri.length() >= sizeof(Buf))
+       abort();
+       

    /* Build the request. We include a keep-alive header only for non-proxy
       requests. This is to tweak old http/1.0 servers that do support keep-alive
       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server 
    /* Build the request. We include a keep-alive header only for non-proxy
       requests. This is to tweak old http/1.0 servers that do support keep-alive
       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server 


@@ -590,10 +617,29 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       and we expect the proxy to do this */
    if (Proxy.empty() == true)
       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
       and we expect the proxy to do this */
    if (Proxy.empty() == true)
       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",

-             Uri.Path.c_str(),ProperHost.c_str());
+             QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());

    else
    else

+   {
+      /* Generate a cache control header if necessary. We place a max
+                cache age on index files, optionally set a no-cache directive
+                and a no-store directive for archives. */

       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
              Itm->Uri.c_str(),ProperHost.c_str());
       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
              Itm->Uri.c_str(),ProperHost.c_str());

+      if (_config->FindB("Acquire::http::No-Cache",false) == true)
+        strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
+      else
+      {
+        if (Itm->IndexFile == true)
+           sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
+                   _config->FindI("Acquire::http::Max-Age",60*60*24));
+        else
+        {
+           if (_config->FindB("Acquire::http::No-Store",false) == true)
+              strcat(Buf,"Cache-Control: no-store\r\n");
+        }       
+      }
+   }
+   

    string Req = Buf;
 
    // Check for a partial file
    string Req = Buf;
 
    // Check for a partial file


@@ -614,12 +660,13 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       }
    }
 
       }
    }
 

-/*   if (ProxyAuth.empty() == false)
-      Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
+   if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
+      Req += string("Proxy-Authorization: Basic ") + 
+          Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";

 
    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 //   cerr << Req << endl;
 
    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 //   cerr << Req << endl;

-   
+

    Out.Read(Req);
 }
                                                                        /*}}}*/
    Out.Read(Req);
 }
                                                                        /*}}}*/


@@ -668,7 +715,7 @@ bool HttpMethod::Go(bool ToFile,ServerState *Srv)
    
    // Select
    struct timeval tv;
    
    // Select
    struct timeval tv;

-   tv.tv_sec = 120;
+   tv.tv_sec = TimeOut;

    tv.tv_usec = 0;
    int Res = 0;
    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
    tv.tv_usec = 0;
    int Res = 0;
    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)


@@ -767,6 +814,7 @@ bool HttpMethod::ServerDie(ServerState *Srv)
    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header && 
        Srv->Encoding != ServerState::Closes)
    {
    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header && 
        Srv->Encoding != ServerState::Closes)
    {

+      Srv->Close();

       if (errno == 0)
         return _error->Error("Error reading from server Remote end closed connection");
       return _error->Errno("read","Error reading from server");
       if (errno == 0)
         return _error->Error("Error reading from server Remote end closed connection");
       return _error->Errno("read","Error reading from server");


@@ -828,6 +876,7 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
       return 5;
 
    FailFile = Queue->DestFile;
       return 5;
 
    FailFile = Queue->DestFile;

+   FailFile.c_str();   // Make sure we don't do a malloc in the signal handler

    FailFd = File->Fd();
    FailTime = Srv->Date;
       
    FailFd = File->Fd();
    FailTime = Srv->Date;
       


@@ -880,6 +929,51 @@ void HttpMethod::SigTerm(int)
    exit(100);
 }
                                                                        /*}}}*/
    exit(100);
 }
                                                                        /*}}}*/

+// HttpMethod::Fetch - Fetch an item                                   /*{{{*/
+// ---------------------------------------------------------------------
+/* This adds an item to the pipeline. We keep the pipeline at a fixed
+   depth. */
+bool HttpMethod::Fetch(FetchItem *)
+{
+   if (Server == 0)
+      return true;
+
+   // Queue the requests
+   int Depth = -1;
+   bool Tail = false;
+   for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
+   {
+      // Make sure we stick with the same server
+      if (Server->Comp(I->Uri) == false)
+        break;
+      if (QueueBack == I)
+        Tail = true;
+      if (Tail == true)
+      {
+        QueueBack = I->Next;
+        SendReq(I,Server->Out);
+        continue;
+      }         
+   }
+   
+   return true;
+};
+                                                                       /*}}}*/
+// HttpMethod::Configuration - Handle a configuration message          /*{{{*/
+// ---------------------------------------------------------------------
+/* We stash the desired pipeline depth */
+bool HttpMethod::Configuration(string Message)
+{
+   if (pkgAcqMethod::Configuration(Message) == false)
+      return false;
+   
+   TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
+   PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
+                                 PipelineDepth);
+   
+   return true;
+}
+                                                                       /*}}}*/

 // HttpMethod::Loop - Main loop                                                /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // HttpMethod::Loop - Main loop                                                /*{{{*/
 // ---------------------------------------------------------------------
 /* */


@@ -888,14 +982,14 @@ int HttpMethod::Loop()
    signal(SIGTERM,SigTerm);
    signal(SIGINT,SigTerm);
    
    signal(SIGTERM,SigTerm);
    signal(SIGINT,SigTerm);
    

-   ServerState *Server = 0;
+   Server = 0;

    
    int FailCounter = 0;
    while (1)
    {
       if (FailCounter >= 2)
       {
    
    int FailCounter = 0;
    while (1)
    {
       if (FailCounter >= 2)
       {

-        Fail("Massive Server Brain Damage");
+        Fail("Massive Server Brain Damage",true);

         FailCounter = 0;
       }
       
         FailCounter = 0;
       }
       


@@ -920,16 +1014,22 @@ int HttpMethod::Loop()
         Server = new ServerState(Queue->Uri,this);
       }
             
         Server = new ServerState(Queue->Uri,this);
       }
             

+      // Reset the pipeline
+      if (Server->ServerFd == -1)
+        QueueBack = Queue;      
+        

       // Connnect to the host
       if (Server->Open() == false)
       {
       // Connnect to the host
       if (Server->Open() == false)
       {

-        Fail();
+        Fail(true);
+        delete Server;
+        Server = 0;

         continue;
       }
         continue;
       }

-      
-      // Queue the request
-      SendReq(Queue,Server->Out);

 
 

+      // Fill the pipeline.
+      Fetch(0);
+      

       // Fetch the next URL header data from the server.
       switch (Server->RunHeaders())
       {
       // Fetch the next URL header data from the server.
       switch (Server->RunHeaders())
       {


@@ -940,7 +1040,7 @@ int HttpMethod::Loop()
         case 2:
         {
            _error->Error("Bad header Data");
         case 2:
         {
            _error->Error("Bad header Data");

-           Fail();
+           Fail(true);

            continue;
         }
         
            continue;
         }
         


@@ -949,12 +1049,12 @@ int HttpMethod::Loop()
         case 1:
         {
            FailCounter++;
         case 1:
         {
            FailCounter++;

-           _error->DumpErrors();
+           _error->Discard();

            Server->Close();
            continue;
         }
       };
            Server->Close();
            continue;
         }
       };

-      
+

       // Decide what to do.
       FetchResult Res;
       Res.Filename = Queue->DestFile;
       // Decide what to do.
       FetchResult Res;
       Res.Filename = Queue->DestFile;