From be4401bfa4a240bbc894e1bfeb1e1e8d63fc7b18 Mon Sep 17 00:00:00 2001 From: Arch Librarian Date: Mon, 20 Sep 2004 16:51:16 +0000 Subject: [PATCH 1/1] New http method Author: jgg Date: 1998-11-01 05:27:29 GMT New http method --- Makefile | 2 +- apt-pkg/acquire-method.cc | 129 +++++- apt-pkg/acquire-method.h | 33 +- apt-pkg/acquire-worker.cc | 6 +- apt-pkg/acquire.cc | 19 +- apt-pkg/acquire.h | 3 +- apt-pkg/contrib/md5.cc | 4 +- apt-pkg/contrib/strutl.cc | 6 +- apt-pkg/contrib/strutl.h | 15 +- doc/Bugs | 33 +- methods/copy.cc | 13 +- methods/file.cc | 11 +- methods/gzip.cc | 14 +- methods/http.cc | 892 ++++++++++++++++++++++++++++++++++++++ methods/http.h | 141 ++++++ methods/makefile | 6 + 16 files changed, 1254 insertions(+), 73 deletions(-) create mode 100644 methods/http.cc create mode 100644 methods/http.h diff --git a/Makefile b/Makefile index af8827067..0c77edd43 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ endif all headers library clean veryclean binary program doc: $(MAKE) -C apt-pkg $@ $(MAKE) -C methods $@ - $(MAKE) -C methods/ftp $@ +# $(MAKE) -C methods/ftp $@ $(MAKE) -C cmdline $@ $(MAKE) -C deity $@ $(MAKE) -C gui $@ diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index bd7dd6779..0629995a0 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire-method.cc,v 1.1 1998/10/30 07:53:35 jgg Exp $ +// $Id: acquire-method.cc,v 1.2 1998/11/01 05:27:30 jgg Exp $ /* ###################################################################### Acquire Method @@ -45,6 +45,8 @@ pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags) if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S)) exit(100); + + SetNonBlock(STDIN_FILENO,true); } /*}}}*/ // AcqMethod::Fail - A fetch has failed /*{{{*/ @@ -65,9 +67,20 @@ void pkgAcqMethod::Fail() void pkgAcqMethod::Fail(string Err) { char S[1024]; - snprintf(S,sizeof(S),"400 URI Failure\nURI: %s\n" - "Message %s\n\n",CurrentURI.c_str(),Err.c_str()); - + if (Queue != 0) + { + snprintf(S,sizeof(S),"400 URI Failure\nURI: %s\n" + "Message: %s\n\n",Queue->Uri.c_str(),Err.c_str()); + + // Dequeue + FetchItem *Tmp = Queue; + Queue = Queue->Next; + delete Tmp; + } + else + snprintf(S,sizeof(S),"400 URI Failure\nURI: \n" + "Message: %s\n\n",Err.c_str()); + if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S)) exit(100); } @@ -75,12 +88,15 @@ void pkgAcqMethod::Fail(string Err) // AcqMethod::URIStart - Indicate a download is starting /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqMethod::URIStart(FetchResult &Res,unsigned long Resume = 0) +void pkgAcqMethod::URIStart(FetchResult &Res) { + if (Queue == 0) + abort(); + char S[1024] = ""; char *End = S; - End += snprintf(S,sizeof(S),"200 URI Start\nURI: %s\n",CurrentURI.c_str()); + End += snprintf(S,sizeof(S),"200 URI Start\nURI: %s\n",Queue->Uri.c_str()); if (Res.Size != 0) End += snprintf(End,sizeof(S) - (End - S),"Size: %u\n",Res.Size); @@ -88,9 +104,9 @@ void pkgAcqMethod::URIStart(FetchResult &Res,unsigned long Resume = 0) End += snprintf(End,sizeof(S) - (End - S),"Last-Modified: %s\n", TimeRFC1123(Res.LastModified).c_str()); - if (Resume != 0) + if (Res.ResumePoint != 0) End += snprintf(End,sizeof(S) - (End - S),"Resume-Point: %u\n", - Resume); + Res.ResumePoint); strcat(End,"\n"); if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S)) @@ -102,10 +118,13 @@ void pkgAcqMethod::URIStart(FetchResult &Res,unsigned long Resume = 0) /* */ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) { + if (Queue == 0) + abort(); + char S[1024] = ""; char *End = S; - End += snprintf(S,sizeof(S),"201 URI Done\nURI: %s\n",CurrentURI.c_str()); + End += snprintf(S,sizeof(S),"201 URI Done\nURI: %s\n",Queue->Uri.c_str()); if (Res.Filename.empty() == false) End += snprintf(End,sizeof(S) - (End - S),"Filename: %s\n",Res.Filename.c_str()); @@ -147,6 +166,11 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) strcat(End,"\n"); if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S)) exit(100); + + // Dequeue + FetchItem *Tmp = Queue; + Queue = Queue->Next; + delete Tmp; } /*}}}*/ // AcqMethod::Configuration - Handle the configuration message /*{{{*/ @@ -186,19 +210,25 @@ bool pkgAcqMethod::Configuration(string Message) // AcqMethod::Run - Run the message engine /*{{{*/ // --------------------------------------------------------------------- /* */ -int pkgAcqMethod::Run() +int pkgAcqMethod::Run(bool Single) { - SetNonBlock(STDIN_FILENO,true); - while (1) { + // Block if the message queue is empty if (Messages.empty() == true) - if (WaitFd(STDIN_FILENO) == false) - return 0; + { + if (Single == false) + if (WaitFd(STDIN_FILENO) == false) + return 0; + } if (ReadMessages(STDIN_FILENO,Messages) == false) return 0; - + + // Single mode exits if the message queue is empty + if (Single == true && Messages.empty() == true) + return 0; + string Message = Messages.front(); Messages.erase(Messages.begin()); @@ -219,12 +249,20 @@ int pkgAcqMethod::Run() break; case 600: - { - CurrentURI = LookupTag(Message,"URI"); - DestFile = LookupTag(Message,"FileName"); - StrToTime(LookupTag(Message,"Last-Modified"),LastModified); - - if (Fetch(Message,CurrentURI) == false) + { + FetchItem *Tmp = new FetchItem; + + Tmp->Uri = LookupTag(Message,"URI"); + Tmp->DestFile = LookupTag(Message,"FileName"); + StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified); + Tmp->Next = 0; + + // Append it to the list + FetchItem **I = &Queue; + for (; *I != 0 && (*I)->Next != 0; I = &(*I)->Next); + *I = Tmp; + + if (Fetch(Tmp) == false) Fail(); break; } @@ -234,6 +272,55 @@ int pkgAcqMethod::Run() return 0; } /*}}}*/ +// AcqMethod::Log - Send a log message /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::Log(const char *Format,...) +{ + string CurrentURI = ""; + if (Queue != 0) + CurrentURI = Queue->Uri; + + va_list args; + va_start(args,Format); + + // sprintf the description + char S[1024]; + unsigned int Len = snprintf(S,sizeof(S),"101 Log\nURI: %s\n" + "Message: ",CurrentURI.c_str()); + + vsnprintf(S+Len,sizeof(S)-Len,Format,args); + strcat(S,"\n\n"); + + if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S)) + exit(100); +} + /*}}}*/ +// AcqMethod::Status - Send a status message /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::Status(const char *Format,...) +{ + string CurrentURI = ""; + if (Queue != 0) + CurrentURI = Queue->Uri; + + va_list args; + va_start(args,Format); + + // sprintf the description + char S[1024]; + unsigned int Len = snprintf(S,sizeof(S),"101 Log\nURI: %s\n" + "Message: ",CurrentURI.c_str()); + + vsnprintf(S+Len,sizeof(S)-Len,Format,args); + strcat(S,"\n\n"); + + if (write(STDOUT_FILENO,S,strlen(S)) != (signed)strlen(S)) + exit(100); +} + /*}}}*/ + // AcqMethod::FetchResult::FetchResult - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 74489913f..e3d18e341 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire-method.h,v 1.1 1998/10/30 07:53:35 jgg Exp $ +// $Id: acquire-method.h,v 1.2 1998/11/01 05:27:32 jgg Exp $ /* ###################################################################### Acquire Method - Method helper class + functions @@ -23,12 +23,16 @@ class pkgAcqMethod { protected: - - string CurrentURI; - string DestFile; - time_t LastModified; - vector Messages; + struct FetchItem + { + FetchItem *Next; + + string Uri; + string DestFile; + time_t LastModified; + }; + struct FetchResult { @@ -37,18 +41,22 @@ class pkgAcqMethod bool IMSHit; string Filename; unsigned long Size; + unsigned long ResumePoint; FetchResult(); }; - + + // State + vector Messages; + FetchItem *Queue; + // Handlers for messages virtual bool Configuration(string Message); - virtual bool Fetch(string Message,URI Get) {return true;}; + virtual bool Fetch(FetchItem *Item) {return true;}; // Outgoing messages void Fail(); void Fail(string Why); -// void Log(const char *Format,...); - void URIStart(FetchResult &Res,unsigned long Resume = 0); + void URIStart(FetchResult &Res); void URIDone(FetchResult &Res,FetchResult *Alt = 0); public: @@ -56,7 +64,10 @@ class pkgAcqMethod enum CnfFlags {SingleInstance = (1<<0), PreScan = (1<<1), Pipeline = (1<<2), SendConfig = (1<<3)}; - int Run(); + void Log(const char *Format,...); + void Status(const char *Format,...); + + int Run(bool Single = false); pkgAcqMethod(const char *Ver,unsigned long Flags = 0); virtual ~pkgAcqMethod() {}; diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 5195b5b8d..a02c6bc04 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire-worker.cc,v 1.8 1998/10/30 07:53:35 jgg Exp $ +// $Id: acquire-worker.cc,v 1.9 1998/11/01 05:27:33 jgg Exp $ /* ###################################################################### Acquire Worker @@ -237,9 +237,9 @@ bool pkgAcquire::Worker::RunMessages() break; } + OwnerQ->ItemDone(Itm); Itm->Owner->Done(Message,atoi(LookupTag(Message,"Size","0").c_str()), LookupTag(Message,"MD5-Hash")); - OwnerQ->ItemDone(Itm); break; } @@ -252,8 +252,8 @@ bool pkgAcquire::Worker::RunMessages() break; } - Itm->Owner->Failed(Message); OwnerQ->ItemDone(Itm); + Itm->Owner->Failed(Message); break; } diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 3ed0e5d28..91b2a7590 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire.cc,v 1.6 1998/10/30 07:53:37 jgg Exp $ +// $Id: acquire.cc,v 1.7 1998/11/01 05:27:34 jgg Exp $ /* ###################################################################### Acquire - File Acquiration @@ -290,21 +290,23 @@ bool pkgAcquire::Run() if (_error->PendingError() == true) break; } - + + // Shut down the acquire bits + Running = false; for (Queue *I = Queues; I != 0; I = I->Next) I->Shutdown(); - Running = false; return _error->PendingError(); } /*}}}*/ -// pkgAcquire::Bump - Called when an item is dequeued /*{{{*/ +// Acquire::Bump - Called when an item is dequeued /*{{{*/ // --------------------------------------------------------------------- /* This routine bumps idle queues in hopes that they will be able to fetch the dequeued item */ void pkgAcquire::Bump() { - + for (Queue *I = Queues; I != 0; I = I->Next) + I->Bump(); } /*}}}*/ @@ -476,3 +478,10 @@ bool pkgAcquire::Queue::Cycle() return Workers->QueueItem(I); } /*}}}*/ +// Queue::Bump - Fetch any pending objects if we are idle /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcquire::Queue::Bump() +{ +} + /*}}}*/ diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 20ce2873f..25fe4b761 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: acquire.h,v 1.6 1998/10/30 07:53:38 jgg Exp $ +// $Id: acquire.h,v 1.7 1998/11/01 05:27:35 jgg Exp $ /* ###################################################################### Acquire - File Acquiration @@ -133,6 +133,7 @@ class pkgAcquire::Queue bool Startup(); bool Shutdown(); bool Cycle(); + void Bump(); Queue(string Name,pkgAcquire *Owner); ~Queue(); diff --git a/apt-pkg/contrib/md5.cc b/apt-pkg/contrib/md5.cc index 046135f5e..671d61d06 100644 --- a/apt-pkg/contrib/md5.cc +++ b/apt-pkg/contrib/md5.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: md5.cc,v 1.1 1998/10/31 05:19:59 jgg Exp $ +// $Id: md5.cc,v 1.2 1998/11/01 05:27:36 jgg Exp $ /* ###################################################################### MD5Sum - MD5 Message Digest Algorithm. @@ -40,7 +40,7 @@ /*}}}*/ // Include Files /*{{{*/ #ifdef __GNUG__ -#pragma interface "apt-pkg/md5.h" +#pragma implementation "apt-pkg/md5.h" #endif #include diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index d6a7143e4..2c3106ceb 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: strutl.cc,v 1.9 1998/10/30 07:53:45 jgg Exp $ +// $Id: strutl.cc,v 1.10 1998/11/01 05:27:37 jgg Exp $ /* ###################################################################### String Util - Some usefull string functions. @@ -609,10 +609,10 @@ bool StrToTime(string Val,time_t &Result) } /*}}}*/ -// URI::URI - Constructor /*{{{*/ +// URI::CopyFrom - Copy from an object /*{{{*/ // --------------------------------------------------------------------- /* This parses the URI into all of its components */ -URI::URI(string U) +void URI::CopyFrom(string U) { string::const_iterator I = U.begin(); diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h index 7e3e47344..f93184a24 100644 --- a/apt-pkg/contrib/strutl.h +++ b/apt-pkg/contrib/strutl.h @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: strutl.h,v 1.8 1998/10/30 07:53:46 jgg Exp $ +// $Id: strutl.h,v 1.9 1998/11/01 05:27:38 jgg Exp $ /* ###################################################################### String Util - These are some usefull string functions @@ -13,8 +13,6 @@ ##################################################################### */ /*}}}*/ -// This is a private header -// Header section: / #ifndef STRUTL_H #define STRUTL_H @@ -40,11 +38,15 @@ bool ReadMessages(int Fd, vector &List); int stringcmp(const char *A,const char *AEnd,const char *B,const char *BEnd); inline int stringcmp(const char *A,const char *AEnd,const char *B) {return stringcmp(A,AEnd,B,B+strlen(B));}; +inline int stringcmp(string A,const char *B) {return stringcmp(A.begin(),A.end(),B,B+strlen(B));}; int stringcasecmp(const char *A,const char *AEnd,const char *B,const char *BEnd); inline int stringcasecmp(const char *A,const char *AEnd,const char *B) {return stringcasecmp(A,AEnd,B,B+strlen(B));}; +inline int stringcasecmp(string A,const char *B) {return stringcasecmp(A.begin(),A.end(),B,B+strlen(B));}; class URI { + void CopyFrom(string From); + public: string Access; @@ -54,9 +56,12 @@ class URI string Path; unsigned int Port; - operator string(); + inline operator string(); + inline operator =(string From) {CopyFrom(From);}; + inline bool empty() {return Access.empty();}; - URI(string Path); + URI(string Path) {CopyFrom(Path);}; + URI() : Port(0) {}; }; #endif diff --git a/doc/Bugs b/doc/Bugs index d6df8d4e2..63a0c9680 100644 --- a/doc/Bugs +++ b/doc/Bugs @@ -2,9 +2,6 @@ #24000: Bug in apt Version: 0.0.17-1bo0 Summary: Couldn't locate an archive source for a package Status: Require Status file. -#27601: srange errors from dselect - Summary: Couldn't locate an archive source - Status: Require status file #26670: apt: apt-get dumps core after checking integrity Summary: Some terminal environments cause APT to crash Win95 telnet and emacs term-in-a-buffer are two at least @@ -29,12 +26,15 @@ Status: Probable that 0.3.x will have support for configuing some parameters #22892: Apt improvements +#28184: apt could be smarted regarding mirrors Summary: Make use of redundant sources Status: 0.3.0 will likely do this, already the information for it is stored. #24799: Some suggestions for the apt method in dselect Summary: Wants to be able to specifiy -d from dselect Status: Likely a APT_OPTIONS enviornment variable will be created, -d can be put there. + There is already an APT_CONFIG in 0.3, APT_OPTIONS may also + appear.. #25104: APT should retry failed downloads Summary: FTP method has problems with busy servers Status: The 0.3.0 ftp method should probably use the configuration mechanism @@ -44,6 +44,14 @@ Summary: FTP method has no controls for firewalls Status: The 0.3.0 ftp method should probably use the configuration mechanism to control this desired behavoir. +#28373: apt package is missing information on ftp.conf + Summary: The man pages have references to several non-existent items, + ftp.conf is only one of them. + Status: Fix the man pages. This certainly will be done in 0.3.0 +#28391: apt-get install without upgrading + Summary: Make install leave the package in the keep state if it is already + installed + Status: Will be implemented in 0.3.0 -- Fixed but unclosed things #25026: apt: Why do you list the packages you're _not_ doing anything to instead of the ones you are? @@ -59,7 +67,14 @@ Status: 0.3.0 has substantially better support for this to the point that it is doable by using a seperate configuration file and the -c option - +#27601: srange errors from dselect + Summary: Couldn't locate an archive source + Status: Require status file + Believed to be fixed in 0.1.9, was not reproducable w/ given + status file +#27841: apt: apt depends on a missing library + Status: New versions of APT in slink have been compiled with libstdc++2.9 + -- Silly things #26592: apt: Problems with ftpd in SunOS 5.6 Summary: SunOS ftpd does not support the SIZE command @@ -74,6 +89,7 @@ Status: Impossible to do without an index file for all source archives. #26019: apt may report wrong line speed #26433: apt: claims to fetch things it doesn't actually fetch (wishlist) +#28778: apt: apt's fetched message is wrong for update of packages files Summary: APT includes the fetch time from the local cache in its calculations Status: Probably will be fixed with new acquire code @@ -93,3 +109,12 @@ Summary: Wants to know what package files were not updated Status: There is no place for this in the current apt-get design, probably won't make the GUI either. +#28172: HTTP Proxy cache refresh should be forced for corrupted packages + Summary: Some problem resulted in a corrupted package + Status: I belive this reflects a deeper problem and the suggested solution + is only a band-aide patch. I intend to close this bug when #24685 + is fixed with a configuration directive. +#27646: Apt: dpkg --merge-avail + Summary: Suggestion to call merge avail after each update operation + Status: Unlikely. The dpkg --print-avail functions should be obsoleted + by the apt-query program which should be written. diff --git a/methods/copy.cc b/methods/copy.cc index e63801f38..94e88e503 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: copy.cc,v 1.4 1998/10/30 07:53:51 jgg Exp $ +// $Id: copy.cc,v 1.5 1998/11/01 05:27:40 jgg Exp $ /* ###################################################################### Copy URI - This method takes a uri like a file: uri and copies it @@ -20,7 +20,7 @@ class CopyMethod : public pkgAcqMethod { - virtual bool Fetch(string Message,URI Get); + virtual bool Fetch(FetchItem *Itm); public: @@ -30,13 +30,14 @@ class CopyMethod : public pkgAcqMethod // CopyMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool CopyMethod::Fetch(string Message,URI Get) +bool CopyMethod::Fetch(FetchItem *Itm) { + URI Get = Itm->Uri; string File = Get.Path; // See if the file exists FileFd From(File,FileFd::ReadOnly); - FileFd To(DestFile,FileFd::WriteEmpty); + FileFd To(Itm->DestFile,FileFd::WriteEmpty); To.EraseOnFailure(); if (_error->PendingError() == true) return false; @@ -59,7 +60,7 @@ bool CopyMethod::Fetch(string Message,URI Get) struct utimbuf TimeBuf; TimeBuf.actime = Buf.st_atime; TimeBuf.modtime = Buf.st_mtime; - if (utime(DestFile.c_str(),&TimeBuf) != 0) + if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0) { To.OpFail(); return _error->Errno("utime","Failed to set modification time"); @@ -68,7 +69,7 @@ bool CopyMethod::Fetch(string Message,URI Get) // Forumulate a result FetchResult Res; Res.Size = Buf.st_size; - Res.Filename = DestFile; + Res.Filename = Itm->DestFile; Res.LastModified = Buf.st_mtime; Res.IMSHit = false; diff --git a/methods/file.cc b/methods/file.cc index 64490749b..0e5e2ebef 100644 --- a/methods/file.cc +++ b/methods/file.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: file.cc,v 1.4 1998/10/30 07:53:52 jgg Exp $ +// $Id: file.cc,v 1.5 1998/11/01 05:27:41 jgg Exp $ /* ###################################################################### File URI method for APT @@ -22,7 +22,7 @@ class FileMethod : public pkgAcqMethod { - virtual bool Fetch(string Message,URI Get); + virtual bool Fetch(FetchItem *Itm); public: @@ -32,8 +32,9 @@ class FileMethod : public pkgAcqMethod // FileMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileMethod::Fetch(string Message,URI Get) +bool FileMethod::Fetch(FetchItem *Itm) { + URI Get = Itm->Uri; string File = Get.Path; FetchResult Res; @@ -45,7 +46,7 @@ bool FileMethod::Fetch(string Message,URI Get) Res.Filename = File; Res.LastModified = Buf.st_mtime; Res.IMSHit = false; - if (LastModified == Buf.st_mtime) + if (Itm->LastModified == Buf.st_mtime) Res.IMSHit = true; } @@ -61,7 +62,7 @@ bool FileMethod::Fetch(string Message,URI Get) AltRes.Filename = File; AltRes.LastModified = Buf.st_mtime; AltRes.IMSHit = false; - if (LastModified == Buf.st_mtime) + if (Itm->LastModified == Buf.st_mtime) AltRes.IMSHit = true; URIDone(Res,&AltRes); diff --git a/methods/gzip.cc b/methods/gzip.cc index acba7fc52..be4f82e6f 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: gzip.cc,v 1.3 1998/10/30 07:53:53 jgg Exp $ +// $Id: gzip.cc,v 1.4 1998/11/01 05:27:43 jgg Exp $ /* ###################################################################### GZip method - Take a file URI in and decompress it into the target @@ -23,7 +23,7 @@ class GzipMethod : public pkgAcqMethod { - virtual bool Fetch(string Message,URI Get); + virtual bool Fetch(FetchItem *Itm); public: @@ -33,11 +33,13 @@ class GzipMethod : public pkgAcqMethod // GzipMethod::Fetch - Decompress the passed URI /*{{{*/ // --------------------------------------------------------------------- /* */ -bool GzipMethod::Fetch(string Message,URI Get) +bool GzipMethod::Fetch(FetchItem *Itm) { + URI Get = Itm->Uri; + // Open the source and destintation files FileFd From(Get.Path,FileFd::ReadOnly); - FileFd To(DestFile,FileFd::WriteEmpty); + FileFd To(Itm->DestFile,FileFd::WriteEmpty); To.EraseOnFailure(); if (_error->PendingError() == true) return false; @@ -90,13 +92,13 @@ bool GzipMethod::Fetch(string Message,URI Get) struct utimbuf TimeBuf; TimeBuf.actime = Buf.st_atime; TimeBuf.modtime = Buf.st_mtime; - if (utime(DestFile.c_str(),&TimeBuf) != 0) + if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0) return _error->Errno("utime","Failed to set modification time"); // Return a Done response FetchResult Res; Res.LastModified = Buf.st_mtime; - Res.Filename = DestFile; + Res.Filename = Itm->DestFile; URIDone(Res); return true; diff --git a/methods/http.cc b/methods/http.cc new file mode 100644 index 000000000..6418a7719 --- /dev/null +++ b/methods/http.cc @@ -0,0 +1,892 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: http.cc,v 1.1 1998/11/01 05:30:47 jgg Exp $ +/* ###################################################################### + + HTTP Aquire Method - This is the HTTP aquire method for APT. + + It uses HTTP/1.1 and many of the fancy options there-in, such as + pipelining, range, if-range and so on. It accepts on the command line + a list of url destination pairs and writes to stdout the status of the + operation as defined in the APT method spec. + + It is based on a doubly buffered select loop. All the requests are + fed into a single output buffer that is constantly fed out the + socket. This provides ideal pipelining as in many cases all of the + requests will fit into a single packet. The input socket is buffered + the same way and fed into the fd for the file. + + This double buffering provides fairly substantial transfer rates, + compared to wget the http method is about 4% faster. Most importantly, + when HTTP is compared with FTP as a protocol the speed difference is + huge. In tests over the internet from two sites to llug (via ATM) this + program got 230k/s sustained http transfer rates. FTP on the other + hand topped out at 170k/s. That combined with the time to setup the + FTP connection makes HTTP a vastly superior protocol. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +// Internet stuff +#include +#include +#include +#include + +#include "http.h" + /*}}}*/ + +// CircleBuf::CircleBuf - Circular input buffer /*{{{*/ +// --------------------------------------------------------------------- +/* */ +CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0) +{ + Buf = new unsigned char[Size]; + Reset(); +} + /*}}}*/ +// CircleBuf::Reset - Reset to the default state /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void CircleBuf::Reset() +{ + InP = 0; + OutP = 0; + StrPos = 0; + MaxGet = (unsigned int)-1; + OutQueue = string(); + if (MD5 != 0) + { + delete MD5; + MD5 = new MD5Summation; + } +}; + /*}}}*/ +// CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/ +// --------------------------------------------------------------------- +/* This fills up the buffer with as much data as is in the FD, assuming it + is non-blocking.. */ +bool CircleBuf::Read(int Fd) +{ + while (1) + { + // Woops, buffer is full + if (InP - OutP == Size) + return true; + + // Write the buffer segment + int Res; + Res = read(Fd,Buf + (InP%Size),LeftRead()); + + if (Res == 0) + return false; + if (Res < 0) + { + if (errno == EAGAIN) + return true; + return false; + } + + if (InP == 0) + gettimeofday(&Start,0); + InP += Res; + } +} + /*}}}*/ +// CircleBuf::Read - Put the string into the buffer /*{{{*/ +// --------------------------------------------------------------------- +/* This will hold the string in and fill the buffer with it as it empties */ +bool CircleBuf::Read(string Data) +{ + OutQueue += Data; + FillOut(); + return true; +} + /*}}}*/ +// CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void CircleBuf::FillOut() +{ + if (OutQueue.empty() == true) + return; + while (1) + { + // Woops, buffer is full + if (InP - OutP == Size) + return; + + // Write the buffer segment + unsigned long Sz = LeftRead(); + if (OutQueue.length() - StrPos < Sz) + Sz = OutQueue.length() - StrPos; + memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz); + + // Advance + StrPos += Sz; + InP += Sz; + if (OutQueue.length() == StrPos) + { + StrPos = 0; + OutQueue = ""; + return; + } + } +} + /*}}}*/ +// CircleBuf::Write - Write from the buffer into a FD /*{{{*/ +// --------------------------------------------------------------------- +/* This empties the buffer into the FD. */ +bool CircleBuf::Write(int Fd) +{ + while (1) + { + FillOut(); + + // Woops, buffer is empty + if (OutP == InP) + return true; + + if (OutP == MaxGet) + return true; + + // Write the buffer segment + int Res; + Res = write(Fd,Buf + (OutP%Size),LeftWrite()); + + if (Res == 0) + return false; + if (Res < 0) + { + if (errno == EAGAIN) + return true; + + return false; + } + + if (MD5 != 0) + MD5->Add(Buf + (OutP%Size),Res); + + OutP += Res; + } +} + /*}}}*/ +// CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/ +// --------------------------------------------------------------------- +/* This copies till the first empty line */ +bool CircleBuf::WriteTillEl(string &Data,bool Single) +{ + // We cheat and assume it is unneeded to have more than one buffer load + for (unsigned long I = OutP; I < InP; I++) + { + if (Buf[I%Size] != '\n') + continue; + for (I++; I < InP && Buf[I%Size] == '\r'; I++); + + if (Single == false) + { + if (Buf[I%Size] != '\n') + continue; + for (I++; I < InP && Buf[I%Size] == '\r'; I++); + } + + if (I > InP) + I = InP; + + Data = ""; + while (OutP < I) + { + unsigned long Sz = LeftWrite(); + if (Sz == 0) + return false; + if (I - OutP < LeftWrite()) + Sz = I - OutP; + Data += string((char *)(Buf + (OutP%Size)),Sz); + OutP += Sz; + } + return true; + } + return false; +} + /*}}}*/ +// CircleBuf::Stats - Print out stats information /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void CircleBuf::Stats() +{ + if (InP == 0) + return; + + struct timeval Stop; + gettimeofday(&Stop,0); +/* float Diff = Stop.tv_sec - Start.tv_sec + + (float)(Stop.tv_usec - Start.tv_usec)/1000000; + clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/ +} + /*}}}*/ + +// ServerState::ServerState - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner), + In(64*1024), Out(1*1024), + ServerName(Srv) +{ + Reset(); +} + /*}}}*/ +// ServerState::Open - Open a connection to the server /*{{{*/ +// --------------------------------------------------------------------- +/* This opens a connection to the server. */ +string LastHost; +in_addr LastHostA; +bool ServerState::Open() +{ + Close(); + + int Port; + string Host; + + if (Proxy.empty() == false) + { + Port = ServerName.Port; + Host = ServerName.Host; + } + else + { + Port = Proxy.Port; + Host = Proxy.Host; + } + + if (LastHost != Host) + { + Owner->Status("Connecting to %s",Host.c_str()); + + // Lookup the host + hostent *Addr = gethostbyname(Host.c_str()); + if (Addr == 0) + return _error->Errno("gethostbyname","Could not lookup host %s",Host.c_str()); + LastHost = Host; + LastHostA = *(in_addr *)(Addr->h_addr_list[0]); + } + + Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA)); + + // Get a socket + if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0) + return _error->Errno("socket","Could not create a socket"); + + // Connect to the server + struct sockaddr_in server; + server.sin_family = AF_INET; + server.sin_port = htons(Port); + server.sin_addr = LastHostA; + if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0) + return _error->Errno("socket","Could not create a socket"); + + SetNonBlock(ServerFd,true); + return true; +} + /*}}}*/ +// ServerState::Close - Close a connection to the server /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool ServerState::Close() +{ + close(ServerFd); + ServerFd = -1; + In.Reset(); + Out.Reset(); + return true; +} + /*}}}*/ +// ServerState::RunHeaders - Get the headers before the data /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool ServerState::RunHeaders() +{ + State = Header; + + Owner->Status("Waiting for file"); + + Major = 0; + Minor = 0; + Result = 0; + Size = 0; + StartPos = 0; + Encoding = Closes; + time(&Date); + + do + { + string Data; + if (In.WriteTillEl(Data) == false) + continue; + + for (string::const_iterator I = Data.begin(); I < Data.end(); I++) + { + string::const_iterator J = I; + for (; J != Data.end() && *J != '\n' && *J != '\r';J++); + if (HeaderLine(string(I,J-I)) == false) + return false; + I = J; + } + return true; + } + while (Owner->Go(false,this) == true); + + return false; +} + /*}}}*/ +// ServerState::RunData - Transfer the data from the socket /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool ServerState::RunData() +{ + State = Data; + + // Chunked transfer encoding is fun.. + if (Encoding == Chunked) + { + while (1) + { + // Grab the block size + bool Last = true; + string Data; + In.Limit(-1); + do + { + if (In.WriteTillEl(Data,true) == true) + break; + } + while ((Last = Owner->Go(false,this)) == true); + + if (Last == false) + return false; + + // See if we are done + unsigned long Len = strtol(Data.c_str(),0,16); + if (Len == 0) + { + In.Limit(-1); + + // We have to remove the entity trailer + Last = true; + do + { + if (In.WriteTillEl(Data,true) == true && Data.length() <= 2) + break; + } + while ((Last = Owner->Go(false,this)) == true); + if (Last == false) + return false; + return true; + } + + // Transfer the block + In.Limit(Len); + while (Owner->Go(true,this) == true) + if (In.IsLimit() == true) + break; + + // Error + if (In.IsLimit() == false) + return false; + + // The server sends an extra new line before the next block specifier.. + In.Limit(-1); + Last = true; + do + { + if (In.WriteTillEl(Data,true) == true) + break; + } + while ((Last = Owner->Go(false,this)) == true); + if (Last == false) + return false; + } + } + else + { + /* Closes encoding is used when the server did not specify a size, the + loss of the connection means we are done */ + if (Encoding == Closes) + In.Limit(-1); + else + In.Limit(Size - StartPos); + + // Just transfer the whole block. + do + { + if (In.IsLimit() == false) + continue; + + In.Limit(-1); + return true; + } + while (Owner->Go(true,this) == true); + } + + return Owner->Flush(this); +} + /*}}}*/ +// ServerState::HeaderLine - Process a header line /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool ServerState::HeaderLine(string Line) +{ + if (Line.empty() == true) + return true; + + // The http server might be trying to do something evil. + if (Line.length() >= MAXLEN) + return _error->Error("Got a single header line over %u chars",MAXLEN); + + string::size_type Pos = Line.find(' '); + if (Pos == string::npos || Pos+1 > Line.length()) + return _error->Error("Bad header line"); + + string Tag = string(Line,0,Pos); + string Val = string(Line,Pos+1); + + if (stringcasecmp(Tag,"HTTP") == 0) + { + // Evil servers return no version + if (Line[4] == '/') + { + if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor, + &Result,Code) != 4) + return _error->Error("The http server sent an invalid reply header"); + } + else + { + Major = 0; + Minor = 9; + if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2) + return _error->Error("The http server sent an invalid reply header"); + } + + return true; + } + + if (stringcasecmp(Tag,"Content-Length:")) + { + if (Encoding == Closes) + Encoding = Stream; + + // The length is already set from the Content-Range header + if (StartPos != 0) + return true; + + if (sscanf(Val.c_str(),"%lu",&Size) != 1) + return _error->Error("The http server sent an invalid Content-Length header"); + return true; + } + + if (stringcasecmp(Tag,"Content-Range:")) + { + if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2) + return _error->Error("The http server sent an invalid Content-Range header"); + if ((unsigned)StartPos > Size) + return _error->Error("This http server has broken range support"); + return true; + } + + if (stringcasecmp(Tag,"Transfer-Encoding:")) + { + if (stringcasecmp(Val,"chunked")) + Encoding = Chunked; + return true; + } + + if (stringcasecmp(Tag,"Last-Modified:")) + { + if (StrToTime(Val,Date) == false) + return _error->Error("Unknown date format"); + return true; + } + + return true; +} + /*}}}*/ + +// HttpMethod::SendReq - Send the HTTP request /*{{{*/ +// --------------------------------------------------------------------- +/* This places the http request in the outbound buffer */ +void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out) +{ + URI Uri = Itm->Uri; + + // The HTTP server expects a hostname with a trailing :port + char Buf[300]; + string ProperHost = Uri.Host; + if (Uri.Port != 0) + { + sprintf(Buf,":%u",Uri.Port); + ProperHost += Buf; + } + + // Build the request + if (Proxy.empty() == true) + sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n", + Uri.Path.c_str(),ProperHost.c_str()); + else + sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n", + Itm->Uri.c_str(),ProperHost.c_str()); + string Req = Buf; + + // Check for a partial file + struct stat SBuf; + if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) + { + // In this case we send an if-range query with a range header + sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1, + TimeRFC1123(SBuf.st_mtime).c_str()); + Req += Buf; + } + else + { + if (Itm->LastModified != 0) + { + sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str()); + Req += Buf; + } + } + +/* if (ProxyAuth.empty() == false) + Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/ + + Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n"; + Out.Read(Req); +} + /*}}}*/ +// HttpMethod::Go - Run a single loop /*{{{*/ +// --------------------------------------------------------------------- +/* This runs the select loop over the server FDs, Output file FDs and + stdin. */ +bool HttpMethod::Go(bool ToFile,ServerState *Srv) +{ + // Server has closed the connection + if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false) + return false; + + fd_set rfds,wfds,efds; + FD_ZERO(&rfds); + FD_ZERO(&wfds); + FD_ZERO(&efds); + + // Add the server + if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1) + FD_SET(Srv->ServerFd,&wfds); + if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1) + FD_SET(Srv->ServerFd,&rfds); + + // Add the file + int FileFD = -1; + if (File != 0) + FileFD = File->Fd(); + + if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1) + FD_SET(FileFD,&wfds); + + // Add stdin + FD_SET(STDIN_FILENO,&rfds); + + // Error Set + if (FileFD != -1) + FD_SET(FileFD,&efds); + if (Srv->ServerFd != -1) + FD_SET(Srv->ServerFd,&efds); + + // Figure out the max fd + int MaxFd = FileFD; + if (MaxFd < Srv->ServerFd) + MaxFd = Srv->ServerFd; + + // Select + struct timeval tv; + tv.tv_sec = 120; + tv.tv_usec = 0; + int Res = 0; + if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0) + return _error->Errno("select","Select failed"); + + if (Res == 0) + { + _error->Error("Connection timed out"); + return ServerDie(Srv); + } + + // Some kind of exception (error) on the sockets, die + if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) || + (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds))) + return _error->Error("Socket Exception"); + + // Handle server IO + if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds)) + { + errno = 0; + if (Srv->In.Read(Srv->ServerFd) == false) + return ServerDie(Srv); + } + + if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds)) + { + errno = 0; + if (Srv->Out.Write(Srv->ServerFd) == false) + return ServerDie(Srv); + } + + // Send data to the file + if (FileFD != -1 && FD_ISSET(FileFD,&wfds)) + { + if (Srv->In.Write(FileFD) == false) + return _error->Errno("write","Error writing to output file"); + } + + // Handle commands from APT + if (FD_ISSET(STDIN_FILENO,&rfds)) + { + if (Run(true) != 0) + exit(100); + } + + return true; +} + /*}}}*/ +// HttpMethod::Flush - Dump the buffer into the file /*{{{*/ +// --------------------------------------------------------------------- +/* This takes the current input buffer from the Server FD and writes it + into the file */ +bool HttpMethod::Flush(ServerState *Srv) +{ + if (File != 0) + { + SetNonBlock(File->Fd(),false); + if (Srv->In.WriteSpace() == false) + return true; + + while (Srv->In.WriteSpace() == true) + { + if (Srv->In.Write(File->Fd()) == false) + return _error->Errno("write","Error writing to file"); + } + + if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes) + return true; + } + return false; +} + /*}}}*/ +// HttpMethod::ServerDie - The server has closed the connection. /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool HttpMethod::ServerDie(ServerState *Srv) +{ + // Dump the buffer to the file + if (Srv->State == ServerState::Data) + { + SetNonBlock(File->Fd(),false); + while (Srv->In.WriteSpace() == true) + { + if (Srv->In.Write(File->Fd()) == false) + return _error->Errno("write","Error writing to the file"); + } + } + + // See if this is because the server finished the data stream + if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header && + Srv->Encoding != ServerState::Closes) + { + if (errno == 0) + return _error->Error("Error reading from server Remote end closed connection"); + return _error->Errno("read","Error reading from server"); + } + else + { + Srv->In.Limit(-1); + + // Nothing left in the buffer + if (Srv->In.WriteSpace() == false) + return false; + + // We may have got multiple responses back in one packet.. + Srv->Close(); + return true; + } + + return false; +} + /*}}}*/ +// HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/ +// --------------------------------------------------------------------- +/* We look at the header data we got back from the server and decide what + to do. Returns + 0 - File is open, + 1 - IMS hit + 3 - Unrecoverable error */ +int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) +{ + // Not Modified + if (Srv->Result == 304) + { + unlink(Queue->DestFile.c_str()); + Res.IMSHit = true; + Res.LastModified = Queue->LastModified; + return 1; + } + + /* We have a reply we dont handle. This should indicate a perm server + failure */ + if (Srv->Result < 200 || Srv->Result >= 300) + { + _error->Error("%u %s",Srv->Result,Srv->Code); + return 3; + } + + // This is some sort of 2xx 'data follows' reply + Res.LastModified = Srv->Date; + Res.Size = Srv->Size; + + // Open the file + delete File; + File = new FileFd(Queue->DestFile,FileFd::WriteAny); + if (_error->PendingError() == true) + return 3; + + // Set the expected size + if (Srv->StartPos >= 0) + { + Res.ResumePoint = Srv->StartPos; + ftruncate(File->Fd(),Srv->StartPos); + } + + // Set the start point + lseek(File->Fd(),0,SEEK_END); + + delete Srv->In.MD5; + Srv->In.MD5 = new MD5Summation; + + // Fill the MD5 Hash if the file is non-empty (resume) + if (Srv->StartPos > 0) + { + lseek(File->Fd(),0,SEEK_SET); + if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false) + { + _error->Errno("read","Problem hashing file"); + return 3; + } + lseek(File->Fd(),0,SEEK_END); + } + + SetNonBlock(File->Fd(),true); + return 0; +} + /*}}}*/ +// HttpMethod::Loop /*{{{*/ +// --------------------------------------------------------------------- +/* */ +int HttpMethod::Loop() +{ + ServerState *Server = 0; + + while (1) + { + // We have no commands, wait for some to arrive + if (Queue == 0) + { + if (WaitFd(STDIN_FILENO) == false) + return 0; + } + + // Run messages + if (Run(true) != 0) + return 100; + + if (Queue == 0) + continue; + + // Connect to the server + if (Server == 0 || Server->Comp(Queue->Uri) == false) + { + delete Server; + Server = new ServerState(Queue->Uri,this); + } + + // Connnect to the host + if (Server->Open() == false) + { + Fail(); + continue; + } + + // Queue the request + SendReq(Queue,Server->In); + + // Handle the header data + if (Server->RunHeaders() == false) + { + Fail(); + continue; + } + + // Decide what to do. + FetchResult Res; + switch (DealWithHeaders(Res,Server)) + { + // Ok, the file is Open + case 0: + { + URIStart(Res); + + // Run the data + if (Server->RunData() == false) + Fail(); + + Res.MD5Sum = Srv->In.MD5->Result(); + delete File; + File = 0; + break; + } + + // IMS hit + case 1: + { + URIDone(Res); + break; + } + + // Hard server error, not found or something + case 3: + { + Fail(); + break; + } + + default: + Fail("Internal error"); + break; + } + } + + return 0; +} + /*}}}*/ + +int main() +{ + HttpMethod Mth; + + return Mth.Loop(); +} diff --git a/methods/http.h b/methods/http.h new file mode 100644 index 000000000..d231aa1f1 --- /dev/null +++ b/methods/http.h @@ -0,0 +1,141 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: http.h,v 1.1 1998/11/01 05:30:47 jgg Exp $ +/* ###################################################################### + + HTTP Aquire Method - This is the HTTP aquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_HTTP_H +#define APT_HTTP_H + +#define MAXLEN 360 + +class HttpMethod; + +class CircleBuf +{ + unsigned char *Buf; + unsigned long Size; + unsigned long InP; + unsigned long OutP; + string OutQueue; + unsigned long StrPos; + unsigned long MaxGet; + struct timeval Start; + + unsigned long LeftRead() + { + unsigned long Sz = Size - (InP - OutP); + if (Sz > Size - (InP%Size)) + Sz = Size - (InP%Size); + return Sz; + } + unsigned long LeftWrite() + { + unsigned long Sz = InP - OutP; + if (InP > MaxGet) + Sz = MaxGet - OutP; + if (Sz > Size - (OutP%Size)) + Sz = Size - (OutP%Size); + return Sz; + } + void FillOut(); + + public: + + MD5Summation *MD5; + + // Read data in + bool Read(int Fd); + bool Read(string Data); + + // Write data out + bool Write(int Fd); + bool WriteTillEl(string &Data,bool Single = false); + + // Control the write limit + void Limit(long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;} + bool IsLimit() {return MaxGet == OutP;}; + void Print() {cout << MaxGet << ',' << OutP << endl;}; + + // Test for free space in the buffer + bool ReadSpace() {return Size - (InP - OutP) > 0;}; + bool WriteSpace() {return InP - OutP > 0;}; + + // Dump everything + void Reset(); + void Stats(); + + CircleBuf(unsigned long Size); + ~CircleBuf() {delete [] Buf;}; +}; + +struct ServerState +{ + // This is the last parsed Header Line + unsigned int Major; + unsigned int Minor; + unsigned int Result; + char Code[MAXLEN]; + + // These are some statistics from the last parsed get line + unsigned long Size; + signed long StartPos; + time_t Date; + enum {Chunked,Stream,Closes} Encoding; + enum {Header, Data} State; + + HttpMethod *Owner; + + // This is the connection itself. Output is data FROM the server + CircleBuf In; + CircleBuf Out; + int ServerFd; + URI ServerName; + + bool HeaderLine(string Line); + bool Comp(URI Other) {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; + void Reset() {Major = 0; Minor = 0; Result = 0; Size = 0; StartPos = 0; + Encoding = Closes; time(&Date); ServerFd = -1;}; + bool RunHeaders(); + bool RunData(); + + bool Open(); + bool Close(); + + ServerState(URI Srv,HttpMethod *Owner); + ~ServerState() {Close();}; +}; + +class HttpMethod : public pkgAcqMethod +{ + void SendReq(FetchItem *Itm,CircleBuf &Out); + bool Go(bool ToFile,ServerState *Srv); + bool Flush(ServerState *Srv); + bool ServerDie(ServerState *Srv); + int DealWithHeaders(FetchResult &Res,ServerState *Srv); + + public: + friend ServerState; + + ServerState *Srv; + int Depth; + FileFd *File; + + int Loop(); + + HttpMethod() : pkgAcqMethod("1.2",SingleInstance | Pipeline | SendConfig) + { + Depth = 0; + Srv = 0; + File = 0; + Depth = 0; + }; +}; + +URI Proxy; + +#endif diff --git a/methods/makefile b/methods/makefile index 1656f5700..b05356bb4 100644 --- a/methods/makefile +++ b/methods/makefile @@ -23,3 +23,9 @@ PROGRAM=gzip SLIBS = -lapt-pkg SOURCE = gzip.cc include $(PROGRAM_H) + +# The http method +PROGRAM=http +SLIBS = -lapt-pkg +SOURCE = http.cc +include $(PROGRAM_H) -- 2.45.2