Fixed dumpavail bug with dup packageS

[apt.git] / methods / http.cc

diff --git a/methods/http.cc b/methods/http.cc
index bfd0fe8edabef538f25c0ffcd9509b764ddbbaeb..b5d26d127754867fefbc5f5926127cb778b5990e 100644 (file)
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/

-// $Id: http.cc,v 1.10 1998/12/07 07:26:27 jgg Exp $
+// $Id: http.cc,v 1.26 1999/02/15 00:26:55 jgg Exp $

 /* ######################################################################
 
    HTTP Aquire Method - This is the HTTP aquire method for APT.
 /* ######################################################################
 
    HTTP Aquire Method - This is the HTTP aquire method for APT.


@@ -38,6 +38,7 @@
 #include <unistd.h>
 #include <signal.h>
 #include <stdio.h>
 #include <unistd.h>
 #include <signal.h>
 #include <stdio.h>

+#include <errno.h>

 
 // Internet stuff
 #include <netinet/in.h>
 
 // Internet stuff
 #include <netinet/in.h>


@@ -51,7 +52,8 @@
 string HttpMethod::FailFile;
 int HttpMethod::FailFd = -1;
 time_t HttpMethod::FailTime = 0;
 string HttpMethod::FailFile;
 int HttpMethod::FailFd = -1;
 time_t HttpMethod::FailTime = 0;

-unsigned long PipelineDepth = 5;
+unsigned long PipelineDepth = 10;
+unsigned long TimeOut = 120;

 
 // CircleBuf::CircleBuf - Circular input buffer                                /*{{{*/
 // ---------------------------------------------------------------------
 
 // CircleBuf::CircleBuf - Circular input buffer                                /*{{{*/
 // ---------------------------------------------------------------------


@@ -246,7 +248,7 @@ void CircleBuf::Stats()
 // ---------------------------------------------------------------------
 /* */
 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 // ---------------------------------------------------------------------
 /* */
 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),

-                        In(64*1024), Out(1*1024),
+                        In(64*1024), Out(4*1024),

                         ServerName(Srv)
 {
    Reset();
                         ServerName(Srv)
 {
    Reset();


@@ -310,7 +312,7 @@ bool ServerState::Open()
 
       // Lookup the host
       hostent *Addr = gethostbyname(Host.c_str());
 
       // Lookup the host
       hostent *Addr = gethostbyname(Host.c_str());

-      if (Addr == 0)
+      if (Addr == 0 || Addr->h_addr_list[0] == 0)

         return _error->Error("Could not resolve '%s'",Host.c_str());
       LastHost = Host;
       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
         return _error->Error("Could not resolve '%s'",Host.c_str());
       LastHost = Host;
       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);


@@ -327,10 +329,30 @@ bool ServerState::Open()
    server.sin_family = AF_INET;
    server.sin_port = htons(Port);
    server.sin_addr = LastHostA;
    server.sin_family = AF_INET;
    server.sin_port = htons(Port);
    server.sin_addr = LastHostA;

-   if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
+   SetNonBlock(ServerFd,true);
+   if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
+       errno != EINPROGRESS)

       return _error->Errno("socket","Could not create a socket");
 
       return _error->Errno("socket","Could not create a socket");
 

-   SetNonBlock(ServerFd,true);
+   /* This implements a timeout for connect by opening the connection
+      nonblocking */
+   fd_set wfds;
+   FD_ZERO(&wfds);
+   FD_SET(ServerFd,&wfds);
+   struct timeval tv;
+   tv.tv_sec = TimeOut;
+   tv.tv_usec = 0;
+   int Res = 0;
+   if ((Res = select(ServerFd+1,0,&wfds,0,&tv)) < 0)
+      return _error->Errno("select","Select failed");
+   if (Res == 0)
+      return _error->Error("Could not connect, connection timed out");
+   unsigned int Err,Len=sizeof(Err);
+   if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
+      return _error->Errno("getsockopt","Failed");
+   if (Err != 0)
+      return _error->Error("Could not connect.");
+   

    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/


@@ -426,7 +448,7 @@ bool ServerState::RunData()
            while ((Last = Owner->Go(false,this)) == true);
            if (Last == false)
               return false;
            while ((Last = Owner->Go(false,this)) == true);
            if (Last == false)
               return false;

-           return true;
+           return !_error->PendingError();

         }
         
         // Transfer the block
         }
         
         // Transfer the block


@@ -468,12 +490,12 @@ bool ServerState::RunData()
            continue;
         
         In.Limit(-1);
            continue;
         
         In.Limit(-1);

-        return true;
+        return !_error->PendingError();

       }
       while (Owner->Go(true,this) == true);
    }
 
       }
       while (Owner->Go(true,this) == true);
    }
 

-   return Owner->Flush(this);
+   return Owner->Flush(this) && !_error->PendingError();

 }
                                                                        /*}}}*/
 // ServerState::HeaderLine - Process a header line                     /*{{{*/
 }
                                                                        /*}}}*/
 // ServerState::HeaderLine - Process a header line                     /*{{{*/


@@ -483,7 +505,7 @@ bool ServerState::HeaderLine(string Line)
 {
    if (Line.empty() == true)
       return true;
 {
    if (Line.empty() == true)
       return true;

-   
+

    // The http server might be trying to do something evil.
    if (Line.length() >= MAXLEN)
       return _error->Error("Got a single header line over %u chars",MAXLEN);
    // The http server might be trying to do something evil.
    if (Line.length() >= MAXLEN)
       return _error->Error("Got a single header line over %u chars",MAXLEN);


@@ -595,7 +617,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       and we expect the proxy to do this */
    if (Proxy.empty() == true)
       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
       and we expect the proxy to do this */
    if (Proxy.empty() == true)
       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",

-             Uri.Path.c_str(),ProperHost.c_str());
+             QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());

    else
    {
       /* Generate a cache control header if necessary. We place a max
    else
    {
       /* Generate a cache control header if necessary. We place a max


@@ -604,7 +626,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
              Itm->Uri.c_str(),ProperHost.c_str());
       if (_config->FindB("Acquire::http::No-Cache",false) == true)
       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
              Itm->Uri.c_str(),ProperHost.c_str());
       if (_config->FindB("Acquire::http::No-Cache",false) == true)

-        strcat(Buf,"Cache-Control: no-cache\r\n");
+        strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");

       else
       {
         if (Itm->IndexFile == true)
       else
       {
         if (Itm->IndexFile == true)


@@ -638,13 +660,13 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       }
    }
 
       }
    }
 

-/*   if (ProxyAuth.empty() == false)
-      Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
+   if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
+      Req += string("Proxy-Authorization: Basic ") + 
+          Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";

 
    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 //   cerr << Req << endl;
 
 
    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 //   cerr << Req << endl;
 

-   cerr << Req << endl;

    Out.Read(Req);
 }
                                                                        /*}}}*/
    Out.Read(Req);
 }
                                                                        /*}}}*/


@@ -693,7 +715,7 @@ bool HttpMethod::Go(bool ToFile,ServerState *Srv)
    
    // Select
    struct timeval tv;
    
    // Select
    struct timeval tv;

-   tv.tv_sec = 120;
+   tv.tv_sec = TimeOut;

    tv.tv_usec = 0;
    int Res = 0;
    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
    tv.tv_usec = 0;
    int Res = 0;
    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)


@@ -792,6 +814,7 @@ bool HttpMethod::ServerDie(ServerState *Srv)
    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header && 
        Srv->Encoding != ServerState::Closes)
    {
    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header && 
        Srv->Encoding != ServerState::Closes)
    {

+      Srv->Close();

       if (errno == 0)
         return _error->Error("Error reading from server Remote end closed connection");
       return _error->Errno("read","Error reading from server");
       if (errno == 0)
         return _error->Error("Error reading from server Remote end closed connection");
       return _error->Errno("read","Error reading from server");


@@ -853,6 +876,7 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
       return 5;
 
    FailFile = Queue->DestFile;
       return 5;
 
    FailFile = Queue->DestFile;

+   FailFile.c_str();   // Make sure we don't do a malloc in the signal handler

    FailFd = File->Fd();
    FailTime = Srv->Date;
       
    FailFd = File->Fd();
    FailTime = Srv->Date;
       


@@ -913,7 +937,7 @@ bool HttpMethod::Fetch(FetchItem *)
 {
    if (Server == 0)
       return true;
 {
    if (Server == 0)
       return true;

-   
+

    // Queue the requests
    int Depth = -1;
    bool Tail = false;
    // Queue the requests
    int Depth = -1;
    bool Tail = false;


@@ -922,12 +946,10 @@ bool HttpMethod::Fetch(FetchItem *)
       // Make sure we stick with the same server
       if (Server->Comp(I->Uri) == false)
         break;
       // Make sure we stick with the same server
       if (Server->Comp(I->Uri) == false)
         break;

-      

       if (QueueBack == I)
         Tail = true;
       if (Tail == true)
       {
       if (QueueBack == I)
         Tail = true;
       if (Tail == true)
       {

-        Depth++;

         QueueBack = I->Next;
         SendReq(I,Server->Out);
         continue;
         QueueBack = I->Next;
         SendReq(I,Server->Out);
         continue;


@@ -937,6 +959,21 @@ bool HttpMethod::Fetch(FetchItem *)
    return true;
 };
                                                                        /*}}}*/
    return true;
 };
                                                                        /*}}}*/

+// HttpMethod::Configuration - Handle a configuration message          /*{{{*/
+// ---------------------------------------------------------------------
+/* We stash the desired pipeline depth */
+bool HttpMethod::Configuration(string Message)
+{
+   if (pkgAcqMethod::Configuration(Message) == false)
+      return false;
+   
+   TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
+   PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
+                                 PipelineDepth);
+   
+   return true;
+}
+                                                                       /*}}}*/

 // HttpMethod::Loop - Main loop                                                /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // HttpMethod::Loop - Main loop                                                /*{{{*/
 // ---------------------------------------------------------------------
 /* */


@@ -952,7 +989,7 @@ int HttpMethod::Loop()
    {
       if (FailCounter >= 2)
       {
    {
       if (FailCounter >= 2)
       {

-        Fail("Massive Server Brain Damage");
+        Fail("Massive Server Brain Damage",true);

         FailCounter = 0;
       }
       
         FailCounter = 0;
       }
       


@@ -977,10 +1014,16 @@ int HttpMethod::Loop()
         Server = new ServerState(Queue->Uri,this);
       }
             
         Server = new ServerState(Queue->Uri,this);
       }
             

+      // Reset the pipeline
+      if (Server->ServerFd == -1)
+        QueueBack = Queue;      
+        

       // Connnect to the host
       if (Server->Open() == false)
       {
       // Connnect to the host
       if (Server->Open() == false)
       {

-        Fail();
+        Fail(true);
+        delete Server;
+        Server = 0;

         continue;
       }
 
         continue;
       }
 


@@ -997,7 +1040,7 @@ int HttpMethod::Loop()
         case 2:
         {
            _error->Error("Bad header Data");
         case 2:
         {
            _error->Error("Bad header Data");

-           Fail();
+           Fail(true);

            continue;
         }
         
            continue;
         }
         


@@ -1006,7 +1049,7 @@ int HttpMethod::Loop()
         case 1:
         {
            FailCounter++;
         case 1:
         {
            FailCounter++;

-           _error->DumpErrors();
+           _error->Discard();

            Server->Close();
            continue;
         }
            Server->Close();
            continue;
         }