ftparchive/cachedb.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: cachedb.cc,v 1.7 2004/05/08 19:41:01 mdz Exp $
   4 /* ######################################################################
   5
   6    CacheDB
   7
   8    Simple uniform interface to a cache database.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/md5.h>
  17 #include <apt-pkg/sha1.h>
  18 #include <apt-pkg/sha2.h>
  19 #include <apt-pkg/strutl.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/fileutl.h>
  22 #include <apt-pkg/debfile.h>
  23 #include <apt-pkg/gpgv.h>
  24 #include <apt-pkg/hashes.h>
  25
  26 #include <netinet/in.h>       // htonl, etc
  27 #include <ctype.h>
  28 #include <stddef.h>
  29 #include <sys/stat.h>
  30 #include <strings.h>
  31
  32 #include "cachedb.h"
  33
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36
  37 // CacheDB::ReadyDB - Ready the DB2                                     /*{{{*/
  38 // ---------------------------------------------------------------------
  39 /* This opens the DB2 file for caching package information */
  40 bool CacheDB::ReadyDB(std::string const &DB)
  41 {
  42    int err;
  43
  44    ReadOnly = _config->FindB("APT::FTPArchive::ReadOnlyDB",false);
  45
  46    // Close the old DB
  47    if (Dbp != 0)
  48       Dbp->close(Dbp,0);
  49
  50    /* Check if the DB was disabled while running and deal with a
  51       corrupted DB */
  52    if (DBFailed() == true)
  53    {
  54       _error->Warning(_("DB was corrupted, file renamed to %s.old"),DBFile.c_str());
  55       rename(DBFile.c_str(),(DBFile+".old").c_str());
  56    }
  57
  58    DBLoaded = false;
  59    Dbp = 0;
  60    DBFile = std::string();
  61
  62    if (DB.empty())
  63       return true;
  64
  65    db_create(&Dbp, NULL, 0);
  66    if ((err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_BTREE,
  67                         (ReadOnly?DB_RDONLY:DB_CREATE),
  68                         0644)) != 0)
  69    {
  70       if (err == DB_OLD_VERSION)
  71       {
  72           _error->Warning(_("DB is old, attempting to upgrade %s"),DBFile.c_str());
  73           err = Dbp->upgrade(Dbp, DB.c_str(), 0);
  74           if (!err)
  75              err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_HASH,
  76                             (ReadOnly?DB_RDONLY:DB_CREATE), 0644);
  77
  78       }
  79       // the database format has changed from DB_HASH to DB_BTREE in
  80       // apt 0.6.44
  81       if (err == EINVAL)
  82       {
  83          _error->Error(_("DB format is invalid. If you upgraded from an older version of apt, please remove and re-create the database."));
  84       }
  85       if (err)
  86       {
  87           Dbp = 0;
  88           return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err));
  89       }
  90    }
  91
  92    DBFile = DB;
  93    DBLoaded = true;
  94    return true;
  95 }
  96                                                                         /*}}}*/
  97 // CacheDB::OpenFile - Open the file                                    /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* */
 100 bool CacheDB::OpenFile()
 101 {
 102    // always close existing file first
 103    CloseFile();
 104
 105    // open a new file
 106    Fd = new FileFd(FileName,FileFd::ReadOnly);
 107    if (_error->PendingError() == true)
 108    {
 109       CloseFile();
 110       return false;
 111    }
 112    return true;
 113 }
 114                                                                         /*}}}*/
 115 // CacheDB::CloseFile - Close the file                                  /*{{{*/
 116 void CacheDB::CloseFile()
 117 {
 118    if(Fd != NULL)
 119    {
 120       delete Fd;
 121       Fd = NULL;
 122    }
 123 }
 124                                                                         /*}}}*/
 125 // CacheDB::OpenDebFile - Open a debfile                                /*{{{*/
 126 bool CacheDB::OpenDebFile()
 127 {
 128    // always close existing file first
 129    CloseDebFile();
 130
 131    // first open the fd, then pass it to the debDebFile
 132    if(OpenFile() == false)
 133       return false;
 134    DebFile = new debDebFile(*Fd);
 135    if (_error->PendingError() == true)
 136       return false;
 137    return true;
 138 }
 139                                                                         /*}}}*/
 140 // CacheDB::CloseDebFile - Close a debfile again                        /*{{{*/
 141 void CacheDB::CloseDebFile()
 142 {
 143    CloseFile();
 144
 145    if(DebFile != NULL)
 146    {
 147       delete DebFile;
 148       DebFile = NULL;
 149    }
 150 }
 151                                                                         /*}}}*/
 152 // CacheDB::GetFileStat - Get stats from the file                       /*{{{*/
 153 // ---------------------------------------------------------------------
 154 /* This gets the size from the database if it's there.  If we need
 155  * to look at the file, also get the mtime from the file. */
 156 bool CacheDB::GetFileStat(bool const &doStat)
 157 {
 158    if ((CurStat.Flags & FlSize) == FlSize && doStat == false)
 159       return true;
 160
 161    /* Get it from the file. */
 162    if (OpenFile() == false)
 163       return false;
 164
 165    // Stat the file
 166    struct stat St;
 167    if (fstat(Fd->Fd(),&St) != 0)
 168    {
 169       CloseFile();
 170       return _error->Errno("fstat",
 171                            _("Failed to stat %s"),FileName.c_str());
 172    }
 173    CurStat.FileSize = St.st_size;
 174    CurStat.mtime = htonl(St.st_mtime);
 175    CurStat.Flags |= FlSize;
 176
 177    return true;
 178 }
 179                                                                         /*}}}*/
 180 // CacheDB::GetCurStat - Set the CurStat variable.                      /*{{{*/
 181 // ---------------------------------------------------------------------
 182 /* Sets the CurStat variable.  Either to 0 if no database is used
 183  * or to the value in the database if one is used */
 184 bool CacheDB::GetCurStat()
 185 {
 186    memset(&CurStat,0,sizeof(CurStat));
 187
 188    if (DBLoaded)
 189    {
 190       /* First see if there is anything about it
 191          in the database */
 192
 193       /* Get the flags (and mtime) */
 194       InitQueryStats();
 195       // Ensure alignment of the returned structure
 196       Data.data = &CurStat;
 197       Data.ulen = sizeof(CurStat);
 198       Data.flags = DB_DBT_USERMEM;
 199       if (Get() == false)
 200       {
 201          CurStat.Flags = 0;
 202       }
 203       CurStat.Flags = ntohl(CurStat.Flags);
 204       CurStat.FileSize = ntohl(CurStat.FileSize);
 205    }
 206    return true;
 207 }
 208                                                                         /*}}}*/
 209 // CacheDB::GetFileInfo - Get all the info about the file               /*{{{*/
 210 // ---------------------------------------------------------------------
 211 bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, bool const &DoContents,
 212                                 bool const &GenContentsOnly, bool const DoSource, unsigned int const DoHashes,
 213                           bool const &checkMtime)
 214 {
 215    this->FileName = FileName;
 216
 217    if (GetCurStat() == false)
 218       return false;
 219    OldStat = CurStat;
 220
 221    if (GetFileStat(checkMtime) == false)
 222       return false;
 223
 224    /* if mtime changed, update CurStat from disk */
 225    if (checkMtime == true && OldStat.mtime != CurStat.mtime)
 226       CurStat.Flags = FlSize;
 227
 228    Stats.Bytes += CurStat.FileSize;
 229    ++Stats.Packages;
 230
 231    if ((DoControl && LoadControl() == false)
 232          || (DoContents && LoadContents(GenContentsOnly) == false)
 233          || (DoSource && LoadSource() == false)
 234          || (DoHashes != 0 && GetHashes(false, DoHashes) == false)
 235       )
 236    {
 237       return false;
 238    }
 239
 240    return true;
 241 }
 242                                                                         /*}}}*/
 243 bool CacheDB::LoadSource()                                              /*{{{*/
 244 {
 245    // Try to read the control information out of the DB.
 246    if ((CurStat.Flags & FlSource) == FlSource)
 247    {
 248       // Lookup the control information
 249       InitQuerySource();
 250       if (Get() == true && Dsc.TakeDsc(Data.data, Data.size) == true)
 251       {
 252             return true;
 253       }
 254       CurStat.Flags &= ~FlSource;
 255    }
 256    if (OpenFile() == false)
 257       return false;
 258
 259    Stats.Misses++;
 260    if (Dsc.Read(FileName) == false)
 261       return false;
 262
 263    if (Dsc.Data == 0)
 264       return _error->Error(_("Failed to read .dsc"));
 265
 266    // Write back the control information
 267    InitQuerySource();
 268    if (Put(Dsc.Data, Dsc.Length) == true)
 269       CurStat.Flags |= FlSource;
 270
 271    return true;
 272 }
 273                                                                         /*}}}*/
 274 // CacheDB::LoadControl - Load Control information                      /*{{{*/
 275 // ---------------------------------------------------------------------
 276 /* */
 277 bool CacheDB::LoadControl()
 278 {
 279    // Try to read the control information out of the DB.
 280    if ((CurStat.Flags & FlControl) == FlControl)
 281    {
 282       // Lookup the control information
 283       InitQueryControl();
 284       if (Get() == true && Control.TakeControl(Data.data,Data.size) == true)
 285             return true;
 286       CurStat.Flags &= ~FlControl;
 287    }
 288
 289    if(OpenDebFile() == false)
 290       return false;
 291
 292    Stats.Misses++;
 293    if (Control.Read(*DebFile) == false)
 294       return false;
 295
 296    if (Control.Control == 0)
 297       return _error->Error(_("Archive has no control record"));
 298
 299    // Write back the control information
 300    InitQueryControl();
 301    if (Put(Control.Control,Control.Length) == true)
 302       CurStat.Flags |= FlControl;
 303    return true;
 304 }
 305                                                                         /*}}}*/
 306 // CacheDB::LoadContents - Load the File Listing                        /*{{{*/
 307 // ---------------------------------------------------------------------
 308 /* */
 309 bool CacheDB::LoadContents(bool const &GenOnly)
 310 {
 311    // Try to read the control information out of the DB.
 312    if ((CurStat.Flags & FlContents) == FlContents)
 313    {
 314       if (GenOnly == true)
 315          return true;
 316
 317       // Lookup the contents information
 318       InitQueryContent();
 319       if (Get() == true)
 320       {
 321          if (Contents.TakeContents(Data.data,Data.size) == true)
 322             return true;
 323       }
 324
 325       CurStat.Flags &= ~FlContents;
 326    }
 327
 328    if(OpenDebFile() == false)
 329       return false;
 330
 331    Stats.Misses++;
 332    if (Contents.Read(*DebFile) == false)
 333       return false;
 334
 335    // Write back the control information
 336    InitQueryContent();
 337    if (Put(Contents.Data,Contents.CurSize) == true)
 338       CurStat.Flags |= FlContents;
 339    return true;
 340 }
 341                                                                         /*}}}*/
 342 // CacheDB::GetHashes - Get the hashs                                   /*{{{*/
 343 static std::string bytes2hex(uint8_t *bytes, size_t length) {
 344    char buf[3];
 345    std::string space;
 346
 347    space.reserve(length*2 + 1);
 348    for (size_t i = 0; i < length; i++) {
 349       snprintf(buf, sizeof(buf), "%02x", bytes[i]);
 350       space.append(buf);
 351    }
 352    return space;
 353 }
 354
 355 static inline unsigned char xdig2num(char const &dig) {
 356    if (isdigit(dig)) return dig - '0';
 357    if ('a' <= dig && dig <= 'f') return dig - 'a' + 10;
 358    if ('A' <= dig && dig <= 'F') return dig - 'A' + 10;
 359    return 0;
 360 }
 361
 362 static void hex2bytes(uint8_t *bytes, const char *hex, int length) {
 363    while (length-- > 0) {
 364       *bytes = 0;
 365       if (isxdigit(hex[0]) && isxdigit(hex[1])) {
 366           *bytes = xdig2num(hex[0]) * 16 + xdig2num(hex[1]);
 367           hex += 2;
 368       }
 369       bytes++;
 370    }
 371 }
 372 bool CacheDB::GetHashes(bool const GenOnly, unsigned int const DoHashes)
 373 {
 374    unsigned int FlHashes = DoHashes & (Hashes::MD5SUM | Hashes::SHA1SUM | Hashes::SHA256SUM | Hashes::SHA512SUM);
 375    HashesList.clear();
 376
 377    if (FlHashes != 0)
 378    {
 379       if (OpenFile() == false)
 380          return false;
 381
 382       Hashes hashes;
 383       if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize, FlHashes) == false)
 384          return false;
 385
 386       HashStringList hl = hashes.GetHashStringList();
 387       for (HashStringList::const_iterator hs = hl.begin(); hs != hl.end(); ++hs)
 388       {
 389          HashesList.push_back(*hs);
 390          if (strcasecmp(hs->HashType().c_str(), "SHA512") == 0)
 391          {
 392             Stats.SHA512Bytes += CurStat.FileSize;
 393             hex2bytes(CurStat.SHA512, hs->HashValue().data(), sizeof(CurStat.SHA512));
 394             CurStat.Flags |= FlSHA512;
 395          }
 396          else if (strcasecmp(hs->HashType().c_str(), "SHA256") == 0)
 397          {
 398             Stats.SHA256Bytes += CurStat.FileSize;
 399             hex2bytes(CurStat.SHA256, hs->HashValue().data(), sizeof(CurStat.SHA256));
 400             CurStat.Flags |= FlSHA256;
 401          }
 402          else if (strcasecmp(hs->HashType().c_str(), "SHA1") == 0)
 403          {
 404             Stats.SHA1Bytes += CurStat.FileSize;
 405             hex2bytes(CurStat.SHA1, hs->HashValue().data(), sizeof(CurStat.SHA1));
 406             CurStat.Flags |= FlSHA1;
 407          }
 408          else if (strcasecmp(hs->HashType().c_str(), "MD5Sum") == 0)
 409          {
 410             Stats.MD5Bytes += CurStat.FileSize;
 411             hex2bytes(CurStat.MD5, hs->HashValue().data(), sizeof(CurStat.MD5));
 412             CurStat.Flags |= FlMD5;
 413          }
 414          else
 415             return _error->Error("Got unknown unrequested hashtype %s", hs->HashType().c_str());
 416       }
 417    }
 418    if (GenOnly == true)
 419       return true;
 420
 421    return HashesList.push_back(HashString("MD5Sum", bytes2hex(CurStat.MD5, sizeof(CurStat.MD5)))) &&
 422       HashesList.push_back(HashString("SHA1", bytes2hex(CurStat.SHA1, sizeof(CurStat.SHA1)))) &&
 423       HashesList.push_back(HashString("SHA256", bytes2hex(CurStat.SHA256, sizeof(CurStat.SHA256)))) &&
 424       HashesList.push_back(HashString("SHA512", bytes2hex(CurStat.SHA512, sizeof(CurStat.SHA512))));
 425 }
 426                                                                         /*}}}*/
 427 // CacheDB::Finish - Write back the cache structure                     /*{{{*/
 428 // ---------------------------------------------------------------------
 429 /* */
 430 bool CacheDB::Finish()
 431 {
 432    // Optimize away some writes.
 433    if (CurStat.Flags == OldStat.Flags &&
 434        CurStat.mtime == OldStat.mtime)
 435       return true;
 436
 437    // Write the stat information
 438    CurStat.Flags = htonl(CurStat.Flags);
 439    CurStat.FileSize = htonl(CurStat.FileSize);
 440    InitQueryStats();
 441    Put(&CurStat,sizeof(CurStat));
 442    CurStat.Flags = ntohl(CurStat.Flags);
 443    CurStat.FileSize = ntohl(CurStat.FileSize);
 444
 445    return true;
 446 }
 447                                                                         /*}}}*/
 448 // CacheDB::Clean - Clean the Database                                  /*{{{*/
 449 // ---------------------------------------------------------------------
 450 /* Tidy the database by removing files that no longer exist at all. */
 451 bool CacheDB::Clean()
 452 {
 453    if (DBLoaded == false)
 454       return true;
 455
 456    /* I'm not sure what VERSION_MINOR should be here.. 2.4.14 certainly
 457       needs the lower one and 2.7.7 needs the upper.. */
 458    DBC *Cursor;
 459    if ((errno = Dbp->cursor(Dbp, NULL, &Cursor, 0)) != 0)
 460       return _error->Error(_("Unable to get a cursor"));
 461
 462    DBT Key;
 463    DBT Data;
 464    memset(&Key,0,sizeof(Key));
 465    memset(&Data,0,sizeof(Data));
 466    while ((errno = Cursor->c_get(Cursor,&Key,&Data,DB_NEXT)) == 0)
 467    {
 468       const char *Colon = (char*)memrchr(Key.data, ':', Key.size);
 469       if (Colon)
 470       {
 471          if (stringcmp(Colon + 1, (char *)Key.data+Key.size,"st") == 0 ||
 472              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cl") == 0 ||
 473              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cs") == 0 ||
 474              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cn") == 0)
 475          {
 476             std::string FileName = std::string((const char *)Key.data,Colon);
 477             if (FileExists(FileName) == true) {
 478                 continue;
 479             }
 480          }
 481       }
 482       Cursor->c_del(Cursor,0);
 483    }
 484    int res = Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL);
 485    if (res < 0)
 486       _error->Warning("compact failed with result %i", res);
 487
 488    if(_config->FindB("Debug::APT::FTPArchive::Clean", false) == true)
 489       Dbp->stat_print(Dbp, 0);
 490
 491
 492    return true;
 493 }
 494                                                                         /*}}}*/