Devel acquire module

Author: jgg
Date: 1998-10-15 06:59:59 GMT
Devel acquire module

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
new file mode 100644 (file)
index 0000000..ccd7259
--- /dev/null
+++ b/apt-pkg/acquire-item.cc
@@ -0,0 +1,85 @@
+// -*- mode: cpp; mode: fold -*-
+// Description                                                         /*{{{*/
+// $Id: acquire-item.cc,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire Item - Item to acquire
+
+   Each item can download to exactly one file at a time. This means you
+   cannot create an item that fetches two uri's to two files at the same 
+   time. The pkgAcqIndex class creates a second class upon instantiation
+   to fetch the other index files because of this.
+   
+   ##################################################################### */
+                                                                       /*}}}*/
+// Include Files                                                       /*{{{*/
+#ifdef __GNUG__
+#pragma implementation "apt-pkg/acquire-item.h"
+#endif
+#include <apt-pkg/acquire-item.h>
+#include <apt-pkg/configuration.h>
+#include <strutl.h>
+                                                                       /*}}}*/
+
+// Acquire::Item::Item - Constructor                                   /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), QueueCounter(0)
+{
+   Owner->Add(this);
+}
+                                                                       /*}}}*/
+// Acquire::Item::~Item - Destructor                                   /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquire::Item::~Item()
+{
+   Owner->Remove(this);
+}
+                                                                       /*}}}*/
+
+// AcqIndex::AcqIndex - Constructor                                    /*{{{*/
+// ---------------------------------------------------------------------
+/* The package file is added to the queue and a second class is 
+   instantiated to fetch the revision file */
+pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location) :
+             Item(Owner), Location(Location)
+{
+   QueueURI(Location->PackagesURI() + ".gz");
+   Description = Location->PackagesInfo();
+   
+   new pkgAcqIndexRel(Owner,Location);
+}
+                                                                       /*}}}*/
+// pkgAcqIndex::ToFile - File to write the download to                 /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+string pkgAcqIndex::ToFile()
+{
+   string PartialDir = _config->FindDir("Dir::State::lists") + "/partial/";
+   
+   return PartialDir + URItoFileName(Location->PackagesURI());
+}
+                                                                       /*}}}*/
+
+// AcqIndexRel::pkgAcqIndexRel - Constructor                           /*{{{*/
+// ---------------------------------------------------------------------
+/* The Release file is added to the queue */
+pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,
+                              const pkgSourceList::Item *Location) :
+                Item(Owner), Location(Location)
+{
+   QueueURI(Location->ReleaseURI());
+   Description = Location->ReleaseInfo();
+}
+                                                                       /*}}}*/
+// AcqIndexRel::ToFile - File to write the download to                 /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+string pkgAcqIndexRel::ToFile()
+{
+   string PartialDir = _config->FindDir("Dir::State::lists") + "/partial/";
+   
+   return PartialDir + URItoFileName(Location->ReleaseURI());
+}
+                                                                       /*}}}*/

diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
new file mode 100644 (file)
index 0000000..6ab8859
--- /dev/null
+++ b/apt-pkg/acquire-item.h
@@ -0,0 +1,76 @@
+// -*- mode: cpp; mode: fold -*-
+// Description                                                         /*{{{*/
+// $Id: acquire-item.h,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire Item - Item to acquire
+
+   When an item is instantiated it will add it self to the local list in
+   the Owner Acquire class. Derived classes will then call QueueURI to 
+   register all the URI's they wish to fetch for at the initial moment.   
+   
+   Two item classes are provided to provide functionality for downloading
+   of Index files and downloading of Packages.
+   
+   ##################################################################### */
+                                                                       /*}}}*/
+#ifndef PKGLIB_ACQUIRE_ITEM_H
+#define PKGLIB_ACQUIRE_ITEM_H
+
+#include <apt-pkg/acquire.h>
+#include <apt-pkg/sourcelist.h>
+
+#ifdef __GNUG__
+#pragma interface "apt-pkg/acquire-item.h"
+#endif 
+
+// Item to acquire
+class pkgAcquire::Item
+{  
+   protected:
+   
+   pkgAcquire *Owner;
+   inline void QueueURI(string URI) {Owner->Enqueue(this,URI);};
+   
+   public:
+
+   unsigned int QueueCounter;
+   string Description;
+   
+   virtual string ToFile() = 0;
+   virtual void Failed() {};
+   
+   Item(pkgAcquire *Owner);
+   virtual ~Item();
+};
+
+// Item class for index files
+class pkgAcqIndex : public pkgAcquire::Item
+{
+   protected:
+   
+   const pkgSourceList::Item *Location;
+   
+   public:
+   
+   virtual string ToFile();
+
+   pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location);
+};
+
+// Item class for index files
+class pkgAcqIndexRel : public pkgAcquire::Item
+{
+   protected:
+   
+   const pkgSourceList::Item *Location;
+   
+   public:
+   
+   virtual string ToFile();
+
+   pkgAcqIndexRel(pkgAcquire *Owner,const pkgSourceList::Item *Location);
+};
+
+
+#endif

diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
new file mode 100644 (file)
index 0000000..34f2072
--- /dev/null
+++ b/apt-pkg/acquire-worker.cc
@@ -0,0 +1,15 @@
+// -*- mode: cpp; mode: fold -*-
+// Description                                                         /*{{{*/
+// $Id: acquire-worker.cc,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire Worker 
+
+   ##################################################################### */
+                                                                       /*}}}*/
+// Include Files                                                       /*{{{*/
+#ifdef __GNUG__
+#pragma implementation "apt-pkg/acquire-worker.h"
+#endif       
+#include <apt-pkg/acquire-worker.h>
+                                                                       /*}}}*/

diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h
new file mode 100644 (file)
index 0000000..f01c935
--- /dev/null
+++ b/apt-pkg/acquire-worker.h
@@ -0,0 +1,39 @@
+// -*- mode: cpp; mode: fold -*-
+// Description                                                         /*{{{*/
+// $Id: acquire-worker.h,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire Worker - Worker process manager
+   
+   ##################################################################### */
+                                                                       /*}}}*/
+#ifndef PKGLIB_ACQUIRE_WORKER_H
+#define PKGLIB_ACQUIRE_WORKER_H
+
+#include <apt-pkg/acquire.h>
+
+#ifdef __GNUG__
+#pragma interface "apt-pkg/acquire-worker.h"
+#endif 
+
+// Interfacing to the method process
+class pkgAcquire::Worker
+{
+   protected:
+   
+   Queue *OwnerQ;
+   MethodConfig *Config;   
+   Worker *Next;
+   
+   friend Queue;
+   
+   public:
+   
+   bool Create();
+   
+   Worker(Queue *OwnerQ);
+   Worker(MethodConfig *Config);
+   ~Worker();
+};
+
+#endif

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
new file mode 100644 (file)
index 0000000..9d8ae99
--- /dev/null
+++ b/apt-pkg/acquire.cc
@@ -0,0 +1,66 @@
+// -*- mode: cpp; mode: fold -*-
+// Description                                                         /*{{{*/
+// $Id: acquire.cc,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire - File Acquiration
+
+   ##################################################################### */
+                                                                       /*}}}*/
+// Include Files                                                       /*{{{*/
+#ifdef __GNUG__
+#pragma implementation "apt-pkg/acquire.h"
+#endif       
+#include <apt-pkg/acquire.h>
+#include <apt-pkg/acquire-item.h>
+#include <apt-pkg/acquire-worker.h>
+                                                                       /*}}}*/
+
+// Acquire::pkgAcquire - Constructor                                   /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquire::pkgAcquire()
+{
+   Queues = 0;
+   Configs = 0;
+}
+                                                                       /*}}}*/
+// Acquire::~pkgAcquire        - Destructor                                    /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+pkgAcquire::~pkgAcquire()
+{
+   while (Items.size() != 0)
+      delete Items[0];
+}
+                                                                       /*}}}*/
+// Acquire::Add - Add a new item                                       /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void pkgAcquire::Add(Item *Itm)
+{
+   Items.push_back(Itm);
+}
+                                                                       /*}}}*/
+// Acquire::Remove - Remove a item                                     /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void pkgAcquire::Remove(Item *Itm)
+{
+   for (vector<Item *>::iterator I = Items.begin(); I < Items.end(); I++)
+   {
+      if (*I == Itm)
+        Items.erase(I);
+   }   
+}
+                                                                       /*}}}*/
+// Acquire::Enqueue - Queue an URI for fetching                                /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+void pkgAcquire::Enqueue(Item *Item,string URI)
+{
+   cout << "Fetching " << URI << endl;
+   cout << "   to " << Item->ToFile() << endl;
+}
+                                                                       /*}}}*/
+

diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
new file mode 100644 (file)
index 0000000..b728d21
--- /dev/null
+++ b/apt-pkg/acquire.h
@@ -0,0 +1,97 @@
+// -*- mode: cpp; mode: fold -*-
+// Description                                                         /*{{{*/
+// $Id: acquire.h,v 1.1 1998/10/15 06:59:59 jgg Exp $
+/* ######################################################################
+
+   Acquire - File Acquiration
+   
+   This module contians the Acquire system. It is responsible for bringing
+   files into the local pathname space. It deals with URIs for files and
+   URI handlers responsible for downloading or finding the URIs.
+   
+   Each file to download is represented by an Acquire::Item class subclassed
+   into a specialization. The Item class can add itself to several URI
+   acquire queues each prioritized by the download scheduler. When the 
+   system is run the proper URI handlers are spawned and the the acquire 
+   queues are fed into the handlers by the schedular until the queues are
+   empty. This allows for an Item to be downloaded from an alternate source
+   if the first try turns out to fail. It also alows concurrent downloading
+   of multiple items from multiple sources as well as dynamic balancing
+   of load between the sources.
+   
+   Schedualing of downloads is done on a first ask first get basis. This
+   preserves the order of the download as much as possible. And means the
+   fastest source will tend to process the largest number of files.
+   
+   Internal methods and queues for performing gzip decompression,
+   md5sum hashing and file copying are provided to allow items to apply
+   a number of transformations to the data files they are working with.
+   
+   ##################################################################### */
+                                                                       /*}}}*/
+#ifndef PKGLIB_ACQUIRE_H
+#define PKGLIB_ACQUIRE_H
+
+#include <vector>
+#include <string>
+
+#ifdef __GNUG__
+#pragma interface "apt-pkg/acquire.h"
+#endif 
+
+class pkgAcquire
+{   
+   public:
+   
+   class Item;
+   class Queue;
+   class Worker;
+   struct MethodConfig;
+   friend Item;
+   
+   protected:
+   
+   vector<Item *> Items;
+   Queue *Queues;
+   MethodConfig *Configs;
+   
+   void Add(Item *Item);
+   void Remove(Item *Item);
+   void Enqueue(Item *Item,string URI);
+   
+   public:
+   
+   pkgAcquire();
+   ~pkgAcquire();
+};
+
+// List of possible items queued for download.
+class pkgAcquire::Queue
+{
+   friend pkgAcquire;
+   Queue *Next;
+   
+   protected:
+   
+   string Access;
+   string URIMatch;
+   
+   vector<Item *> Items;
+   
+   public:
+};
+
+// Configuration information from each method
+struct pkgAcquire::MethodConfig
+{
+   string Access;
+
+   string Version;
+   bool SingleInstance;
+   bool PreScan;
+   
+   MethodConfig();
+   ~MethodConfig();
+};
+
+#endif

diff --git a/apt-pkg/makefile b/apt-pkg/makefile
index 2c33400d7b6fc620b3dcad96eb568fe8b7b6ac89..5bf6f9a8b1732ab1564d1220a9940506cdc892e5 100644 (file)
--- a/apt-pkg/makefile
+++ b/apt-pkg/makefile
@@ -22,8 +22,9 @@ SOURCE = contrib/mmap.cc contrib/error.cc contrib/strutl.cc \
 # Source code for the main library
 SOURCE+= pkgcache.cc version.cc fileutl.cc pkgcachegen.cc depcache.cc \
          orderlist.cc tagfile.cc sourcelist.cc packagemanager.cc \
 # Source code for the main library
 SOURCE+= pkgcache.cc version.cc fileutl.cc pkgcachegen.cc depcache.cc \
          orderlist.cc tagfile.cc sourcelist.cc packagemanager.cc \

-        pkgrecords.cc algorithms.cc init.cc templates.cc
-        
+        pkgrecords.cc algorithms.cc acquire.cc acquire-item.cc \
+        acquire-worker.cc init.cc templates.cc
+

 # Source code for the debian specific components        
 SOURCE+= deb/deblistparser.cc deb/debrecords.cc
 
 # Source code for the debian specific components        
 SOURCE+= deb/deblistparser.cc deb/debrecords.cc
 


@@ -31,7 +32,9 @@ SOURCE+= deb/deblistparser.cc deb/debrecords.cc
 HEADERS = algorithms.h depcache.h mmap.h pkgcachegen.h cacheiterators.h \
           error.h orderlist.h sourcelist.h configuration.h fileutl.h \
           packagemanager.h tagfile.h deblistparser.h init.h pkgcache.h \
 HEADERS = algorithms.h depcache.h mmap.h pkgcachegen.h cacheiterators.h \
           error.h orderlist.h sourcelist.h configuration.h fileutl.h \
           packagemanager.h tagfile.h deblistparser.h init.h pkgcache.h \

-          version.h progress.h pkgrecords.h debrecords.h cmndline.h
+          version.h progress.h pkgrecords.h debrecords.h cmndline.h \
+         acquire.h acquire-worker.h acquire-item.h
+

 HEADERS := $(addprefix apt-pkg/,$(HEADERS))
 
 # Private header files
 HEADERS := $(addprefix apt-pkg/,$(HEADERS))
 
 # Private header files

diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 567628ef5dbef6f6f3ffc94b80522990ae2b65b3..7227f15ebb44e74280de50c9cc4f587ac66a67c2 100644 (file)
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/

-// $Id: pkgcachegen.cc,v 1.18 1998/10/02 04:39:47 jgg Exp $
+// $Id: pkgcachegen.cc,v 1.19 1998/10/15 07:00:00 jgg Exp $

 /* ######################################################################
    
    Package Cache Generator - Generator for the cache structure.
 /* ######################################################################
    
    Package Cache Generator - Generator for the cache structure.


@@ -487,10 +487,31 @@ bool pkgPkgCacheCheck(string CacheFile)
       return false;
    }
 
       return false;
    }
 

+   // Status files that must be in the cache
+   string Status[3];
+   Status[0] = _config->FindDir("Dir::State::xstatus");
+   Status[1]= _config->FindDir("Dir::State::userstatus");
+   Status[2] = _config->FindDir("Dir::State::status");
+   

    // Cheack each file
    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
    // Cheack each file
    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)

+   {

       if (F.IsOk() == false)
         return false;
       if (F.IsOk() == false)
         return false;

+      
+      // See if this is one of the status files
+      for (int I = 0; I != 3; I++)
+        if (F.FileName() == Status[I])
+           Status[I] = string();
+   }
+   
+   // Make sure all the status files are loaded.
+   for (int I = 0; I != 3; I++)
+   {
+      if (Status[I].empty() == false && FileExists(Status[I]) == true)
+        return false;
+   }   
+   

    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/

diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index 6fd5e1eb54ea5ff32e8358a3473d64c9c5e4485d..c9d0285a9a4c0916413ecbb82029c551b97c79e7 100644 (file)
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/

-// $Id: sourcelist.cc,v 1.5 1998/07/26 04:49:33 jgg Exp $
+// $Id: sourcelist.cc,v 1.6 1998/10/15 07:00:01 jgg Exp $

 /* ######################################################################
 
    List of Sources
 /* ######################################################################
 
    List of Sources


@@ -195,6 +195,48 @@ string pkgSourceList::Item::PackagesInfo() const
    return Res;
 }
                                                                        /*}}}*/
    return Res;
 }
                                                                        /*}}}*/

+// SourceList::Item::ReleaseURI - Returns a URI to the release file    /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+string pkgSourceList::Item::ReleaseURI() const
+{
+   string Res;
+   switch (Type)
+   {
+      case Deb:
+      if (Dist[Dist.size() - 1] == '/')
+        Res = URI + Dist;
+      else
+        Res = URI + "dists/" + Dist + '/' + Section +
+        "/binary-" + _config->Find("APT::Architecture") + '/';
+      
+      Res += "Release";
+      break;
+   };
+   return Res;
+}
+                                                                       /*}}}*/
+// SourceList::Item::ReleaseInfo - Shorter version of the URI          /*{{{*/
+// ---------------------------------------------------------------------
+/* This is a shorter version that is designed to be < 60 chars or so */
+string pkgSourceList::Item::ReleaseInfo() const
+{
+   string Res;
+   switch (Type)
+   {
+      case Deb:
+      Res += SiteOnly(URI) + ' ';
+      if (Dist[Dist.size() - 1] == '/')
+        Res += Dist;
+      else
+        Res += Dist + '/' + Section;
+      
+      Res += " Release";
+      break;
+   };
+   return Res;
+}
+                                                                       /*}}}*/

 // SourceList::Item::ArchiveInfo - Shorter version of the archive spec /*{{{*/
 // ---------------------------------------------------------------------
 /* This is a shorter version that is designed to be < 60 chars or so */
 // SourceList::Item::ArchiveInfo - Shorter version of the archive spec /*{{{*/
 // ---------------------------------------------------------------------
 /* This is a shorter version that is designed to be < 60 chars or so */

diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h
index 78d8a22a9666f48d0b37da35a9a21d0c5b7e0aa7..0ed77b3018649aafdcca6dc92bd608d60844efff 100644 (file)
--- a/apt-pkg/sourcelist.h
+++ b/apt-pkg/sourcelist.h
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/

-// $Id: sourcelist.h,v 1.5 1998/07/19 21:24:15 jgg Exp $
+// $Id: sourcelist.h,v 1.6 1998/10/15 07:00:02 jgg Exp $

 /* ######################################################################
 
    SourceList - Manage a list of sources
 /* ######################################################################
 
    SourceList - Manage a list of sources


@@ -47,6 +47,8 @@ class pkgSourceList
       bool SetURI(string S);
       string PackagesURI() const;
       string PackagesInfo() const;      
       bool SetURI(string S);
       string PackagesURI() const;
       string PackagesInfo() const;      

+      string ReleaseURI() const;
+      string ReleaseInfo() const;      

       string SiteOnly(string URI) const;
       string ArchiveInfo(pkgCache::VerIterator Ver) const;
       string ArchiveURI(string File) const;
       string SiteOnly(string URI) const;
       string ArchiveInfo(pkgCache::VerIterator Ver) const;
       string ArchiveURI(string File) const;


@@ -67,7 +69,7 @@ class pkgSourceList
    inline const_iterator end() const {return List.end();};
    inline unsigned int size() const {return List.size();};
    inline bool empty() const {return List.empty();};
    inline const_iterator end() const {return List.end();};
    inline unsigned int size() const {return List.size();};
    inline bool empty() const {return List.empty();};

-   
+

    pkgSourceList();
    pkgSourceList(string File);   
 };
    pkgSourceList();
    pkgSourceList(string File);   
 };