improve pkgTagSection scanning and parsing

[apt.git] / apt-pkg / tagfile.cc

diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc
index 91d176e3ca3a857f03f70ab725a72fda7f489590..52f4da2d5864fe5dd459d568b474176e15b20931 100644 (file)
--- a/apt-pkg/tagfile.cc
+++ b/apt-pkg/tagfile.cc
@@ -47,16 +47,43 @@ public:
    unsigned long long Size;
 };
 
    unsigned long long Size;
 };
 

+static unsigned long AlphaHash(const char *Text, size_t Length)                /*{{{*/
+{
+   /* This very simple hash function for the last 8 letters gives
+      very good performance on the debian package files */
+   if (Length > 8)
+   {
+    Text += (Length - 8);
+    Length = 8;
+   }
+   unsigned long Res = 0;
+   for (size_t i = 0; i < Length; ++i)
+      Res = ((unsigned long)(Text[i]) & 0xDF) ^ (Res << 1);
+   return Res & 0xFF;
+}
+                                                                       /*}}}*/
+

 // TagFile::pkgTagFile - Constructor                                   /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long long Size)
 // TagFile::pkgTagFile - Constructor                                   /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long long Size)

+   : d(NULL)
+{
+   Init(pFd, Size);
+}
+
+void pkgTagFile::Init(FileFd *pFd,unsigned long long Size)

 {
    /* The size is increased by 4 because if we start with the Size of the
       filename we need to try to read 1 char more to see an EOF faster, 1
       char the end-pointer can be on and maybe 2 newlines need to be added
       to the end of the file -> 4 extra chars */
    Size += 4;
 {
    /* The size is increased by 4 because if we start with the Size of the
       filename we need to try to read 1 char more to see an EOF faster, 1
       char the end-pointer can be on and maybe 2 newlines need to be added
       to the end of the file -> 4 extra chars */
    Size += 4;

+   if(d != NULL)
+   {
+      free(d->Buffer);
+      delete d;
+   }

    d = new pkgTagFilePrivate(pFd, Size);
 
    if (d->Fd.IsOpen() == false)
    d = new pkgTagFilePrivate(pFd, Size);
 
    if (d->Fd.IsOpen() == false)


@@ -128,18 +155,23 @@ bool pkgTagFile::Resize(unsigned long long const newSize)
  */
 bool pkgTagFile::Step(pkgTagSection &Tag)
 {
  */
 bool pkgTagFile::Step(pkgTagSection &Tag)
 {

-   while (Tag.Scan(d->Start,d->End - d->Start) == false)
+   if(Tag.Scan(d->Start,d->End - d->Start) == false)

    {
    {

-      if (Fill() == false)
-        return false;
-      
-      if(Tag.Scan(d->Start,d->End - d->Start))
-        break;
+      do
+      {
+        if (Fill() == false)
+           return false;
+
+        if(Tag.Scan(d->Start,d->End - d->Start, false))
+           break;

 
 

-      if (Resize() == false)
-        return _error->Error(_("Unable to parse package file %s (1)"),
-                              d->Fd.Name().c_str());
+        if (Resize() == false)
+           return _error->Error(_("Unable to parse package file %s (1)"),
+                 d->Fd.Name().c_str());
+
+      } while (Tag.Scan(d->Start,d->End - d->Start, false) == false);

    }
    }

+

    d->Start += Tag.size();
    d->iOffset += Tag.size();
 
    d->Start += Tag.size();
    d->iOffset += Tag.size();
 


@@ -233,7 +265,7 @@ bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset)
    if (Fill() == false)
       return false;
    
    if (Fill() == false)
       return false;
    

-   if (Tag.Scan(d->Start, d->End - d->Start) == false)
+   if (Tag.Scan(d->Start, d->End - d->Start, false) == false)

       return _error->Error(_("Unable to parse package file %s (2)"),d->Fd.Name().c_str());
    
    return true;
       return _error->Error(_("Unable to parse package file %s (2)"),d->Fd.Name().c_str());
    
    return true;


@@ -243,27 +275,46 @@ bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset)
 // ---------------------------------------------------------------------
 /* */
 pkgTagSection::pkgTagSection()
 // ---------------------------------------------------------------------
 /* */
 pkgTagSection::pkgTagSection()

-   : Section(0), TagCount(0), d(NULL), Stop(0)
+   : Section(0), d(NULL), Stop(0)

 {
 {

-   memset(&Indexes, 0, sizeof(Indexes));
-   memset(&AlphaIndexes, 0, sizeof(AlphaIndexes));
+   memset(&LookupTable, 0, sizeof(LookupTable));

 }
                                                                        /*}}}*/
 // TagSection::Scan - Scan for the end of the header information       /*{{{*/
 }
                                                                        /*}}}*/
 // TagSection::Scan - Scan for the end of the header information       /*{{{*/

-// ---------------------------------------------------------------------
-/* This looks for the first double new line in the data stream.
-   It also indexes the tags in the section. */
-bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength)
+bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const Restart)

 {
 {

+   Section = Start;

    const char *End = Start + MaxLength;
    const char *End = Start + MaxLength;

-   Stop = Section = Start;
-   memset(AlphaIndexes,0,sizeof(AlphaIndexes));
+
+   if (Restart == false && Tags.empty() == false)
+   {
+      Stop = Section + Tags.back().StartTag;
+      if (End <= Stop)
+        return false;
+      Stop = (const char *)memchr(Stop,'\n',End - Stop);
+      if (Stop == NULL)
+        return false;
+      ++Stop;
+   }
+   else
+   {
+      Stop = Section;
+      if (Tags.empty() == false)
+      {
+        memset(&LookupTable, 0, sizeof(LookupTable));
+        Tags.clear();
+      }
+      Tags.reserve(0x100);
+   }
+   size_t TagCount = Tags.size();

 
    if (Stop == 0)
       return false;
 
 
    if (Stop == 0)
       return false;
 

-   TagCount = 0;
-   while (TagCount+1 < sizeof(Indexes)/sizeof(Indexes[0]) && Stop < End)
+   TagData lastTagData(0);
+   lastTagData.EndTag = 0;
+   unsigned long lastTagHash = 0;
+   while (Stop < End)

    {
       TrimRecord(true,End);
 
    {
       TrimRecord(true,End);
 


@@ -275,12 +326,39 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength)
       // Start a new index and add it to the hash
       if (isspace(Stop[0]) == 0)
       {
       // Start a new index and add it to the hash
       if (isspace(Stop[0]) == 0)
       {

-        Indexes[TagCount++] = Stop - Section;
-        AlphaIndexes[AlphaHash(Stop,End)] = TagCount;
+        // store the last found tag
+        if (lastTagData.EndTag != 0)
+        {
+           if (LookupTable[lastTagHash] != 0)
+              lastTagData.NextInBucket = LookupTable[lastTagHash];
+           LookupTable[lastTagHash] = TagCount;
+           Tags.push_back(lastTagData);
+        }
+
+        ++TagCount;
+        lastTagData = TagData(Stop - Section);
+        // find the colon separating tag and value
+        char const * Colon = (char const *) memchr(Stop, ':', End - Stop);
+        if (Colon == NULL)
+           return false;
+        // find the end of the tag (which might or might not be the colon)
+        char const * EndTag = Colon;
+        --EndTag;
+        for (; EndTag > Stop && isspace(*EndTag) != 0; --EndTag)
+           ;
+        ++EndTag;
+        lastTagData.EndTag = EndTag - Section;
+        lastTagHash = AlphaHash(Stop, EndTag - Stop);
+        // find the beginning of the value
+        Stop = Colon + 1;
+        for (; isspace(*Stop) != 0; ++Stop);
+        if (Stop >= End)
+           return false;
+        lastTagData.StartValue = Stop - Section;

       }
 
       Stop = (const char *)memchr(Stop,'\n',End - Stop);
       }
 
       Stop = (const char *)memchr(Stop,'\n',End - Stop);

-      
+

       if (Stop == 0)
         return false;
 
       if (Stop == 0)
         return false;
 


@@ -291,7 +369,16 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength)
       // Double newline marks the end of the record
       if (Stop+1 < End && Stop[1] == '\n')
       {
       // Double newline marks the end of the record
       if (Stop+1 < End && Stop[1] == '\n')
       {

-        Indexes[TagCount] = Stop - Section;
+        if (lastTagData.EndTag != 0)
+        {
+           if (LookupTable[lastTagHash] != 0)
+              lastTagData.NextInBucket = LookupTable[lastTagHash];
+           LookupTable[lastTagHash] = TagCount;
+           Tags.push_back(lastTagData);
+        }
+
+        TagData const td(Stop - Section);
+        Tags.push_back(td);

         TrimRecord(false,End);
         return true;
       }
         TrimRecord(false,End);
         return true;
       }


@@ -320,8 +407,8 @@ void pkgTagSection::Trim()
    for (; Stop > Section + 2 && (Stop[-2] == '\n' || Stop[-2] == '\r'); Stop--);
 }
                                                                        /*}}}*/
    for (; Stop > Section + 2 && (Stop[-2] == '\n' || Stop[-2] == '\r'); Stop--);
 }
                                                                        /*}}}*/

-// TagSection::Exists - return True if a tag exists                    /*{{{*/
-bool pkgTagSection::Exists(const char* const Tag)
+// TagSection::Exists - return True if a tag exists                    /*{{{*/
+bool pkgTagSection::Exists(const char* const Tag) const

 {
    unsigned int tmp;
    return Find(Tag, tmp);
 {
    unsigned int tmp;
    return Find(Tag, tmp);


@@ -332,73 +419,43 @@ bool pkgTagSection::Exists(const char* const Tag)
 /* This searches the section for a tag that matches the given string. */
 bool pkgTagSection::Find(const char *Tag,unsigned int &Pos) const
 {
 /* This searches the section for a tag that matches the given string. */
 bool pkgTagSection::Find(const char *Tag,unsigned int &Pos) const
 {

-   unsigned int Length = strlen(Tag);
-   unsigned int I = AlphaIndexes[AlphaHash(Tag)];
-   if (I == 0)
+   size_t const Length = strlen(Tag);
+   unsigned int Bucket = LookupTable[AlphaHash(Tag, Length)];
+   if (Bucket == 0)

       return false;
       return false;

-   I--;
-   
-   for (unsigned int Counter = 0; Counter != TagCount; Counter++, 
-       I = (I+1)%TagCount)
+
+   for (; Bucket != 0; Bucket = Tags[Bucket - 1].NextInBucket)

    {
    {

-      const char *St;
-      St = Section + Indexes[I];
-      if (strncasecmp(Tag,St,Length) != 0)
+      if ((Tags[Bucket - 1].EndTag - Tags[Bucket - 1].StartTag) != Length)

         continue;
 
         continue;
 

-      // Make sure the colon is in the right place
-      const char *C = St + Length;
-      for (; isspace(*C) != 0; C++);
-      if (*C != ':')
+      char const * const St = Section + Tags[Bucket - 1].StartTag;
+      if (strncasecmp(Tag,St,Length) != 0)

         continue;
         continue;

-      Pos = I;
+
+      Pos = Bucket - 1;

       return true;
    }
 
    Pos = 0;
    return false;
 }
       return true;
    }
 
    Pos = 0;
    return false;
 }

-                                                                       /*}}}*/
-// TagSection::Find - Locate a tag                                     /*{{{*/
-// ---------------------------------------------------------------------
-/* This searches the section for a tag that matches the given string. */

 bool pkgTagSection::Find(const char *Tag,const char *&Start,
                         const char *&End) const
 {
 bool pkgTagSection::Find(const char *Tag,const char *&Start,
                         const char *&End) const
 {

-   unsigned int Length = strlen(Tag);
-   unsigned int I = AlphaIndexes[AlphaHash(Tag)];
-   if (I == 0)
+   unsigned int Pos;
+   if (Find(Tag, Pos) == false)

       return false;
       return false;

-   I--;
-   
-   for (unsigned int Counter = 0; Counter != TagCount; Counter++, 
-       I = (I+1)%TagCount)
-   {
-      const char *St;
-      St = Section + Indexes[I];
-      if (strncasecmp(Tag,St,Length) != 0)
-        continue;
-      
-      // Make sure the colon is in the right place
-      const char *C = St + Length;
-      for (; isspace(*C) != 0; C++);
-      if (*C != ':')
-        continue;

 
 

-      // Strip off the gunk from the start end
-      Start = C;
-      End = Section + Indexes[I+1];
-      if (Start >= End)
-        return _error->Error("Internal parsing error");
-      
-      for (; (isspace(*Start) != 0 || *Start == ':') && Start < End; Start++);
-      for (; isspace(End[-1]) != 0 && End > Start; End--);
-      
-      return true;
-   }
-   
-   Start = End = 0;
-   return false;
+   Start = Section + Tags[Pos].StartValue;
+   // Strip off the gunk from the end
+   End = Section + Tags[Pos + 1].StartTag;
+   if (unlikely(Start > End))
+      return _error->Error("Internal parsing error");
+
+   for (; isspace(End[-1]) != 0 && End > Start; --End);
+
+   return true;

 }
                                                                        /*}}}*/
 // TagSection::FindS - Find a string                                   /*{{{*/
 }
                                                                        /*}}}*/
 // TagSection::FindS - Find a string                                   /*{{{*/


@@ -493,6 +550,13 @@ bool pkgTagSection::FindFlag(unsigned long &Flags, unsigned long Flag,
    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/

+APT_PURE unsigned int pkgTagSection::Count() const {                   /*{{{*/
+   if (Tags.empty() == true)
+      return 0;
+   // the last element is just marking the end and isn't a real one
+   return Tags.size() - 1;
+}
+                                                                       /*}}}*/

 // TFRewrite - Rewrite a control record                                        /*{{{*/
 // ---------------------------------------------------------------------
 /* This writes the control record to stdout rewriting it as necessary. The
 // TFRewrite - Rewrite a control record                                        /*{{{*/
 // ---------------------------------------------------------------------
 /* This writes the control record to stdout rewriting it as necessary. The