From dddb1fc74cc15f34219c2b4eda474245335cdc90 Mon Sep 17 00:00:00 2001 From: Josiah Gaskin Date: Mon, 6 Jun 2011 17:00:35 -0700 Subject: [PATCH] Added Caching for PreProcessed PNGs Added a cache management system for pre-processed PNG files along with unit tests. The cache system will be used if the --no-crunch flag is passed to AAPT during the package phase. The cache can be updated by a call to 'aapt crunch' (see usage statement). Also put in benchmarking code. Change-Id: I58271fb2ee2f5f9075fd74d4ff6f15e7afabd05c --- Android.mk | 2 + Bundle.h | 12 +++- CacheUpdater.h | 107 +++++++++++++++++++++++++++++++ Command.cpp | 22 +++++++ CrunchCache.cpp | 104 +++++++++++++++++++++++++++++++ CrunchCache.h | 102 ++++++++++++++++++++++++++++++ DirectoryWalker.h | 98 +++++++++++++++++++++++++++++ FileFinder.cpp | 92 +++++++++++++++++++++++++++ FileFinder.h | 78 +++++++++++++++++++++++ Images.cpp | 121 ++++++++++++++++++++++++++++++++++++ Images.h | 10 ++- Main.cpp | 21 ++++++- Main.h | 11 ++++ Package.cpp | 9 +++ Resource.cpp | 52 +++++++++++++--- tests/CrunchCache_test.cpp | 97 +++++++++++++++++++++++++++++ tests/FileFinder_test.cpp | 101 ++++++++++++++++++++++++++++++ tests/MockCacheUpdater.h | 40 ++++++++++++ tests/MockDirectoryWalker.h | 85 +++++++++++++++++++++++++ tests/MockFileFinder.h | 55 ++++++++++++++++ 20 files changed, 1206 insertions(+), 13 deletions(-) create mode 100644 CacheUpdater.h create mode 100644 CrunchCache.cpp create mode 100644 CrunchCache.h create mode 100644 DirectoryWalker.h create mode 100644 FileFinder.cpp create mode 100644 FileFinder.h create mode 100644 tests/CrunchCache_test.cpp create mode 100644 tests/FileFinder_test.cpp create mode 100644 tests/MockCacheUpdater.h create mode 100644 tests/MockDirectoryWalker.h create mode 100644 tests/MockFileFinder.h diff --git a/Android.mk b/Android.mk index 094b7db..e507fb9 100644 --- a/Android.mk +++ b/Android.mk @@ -13,6 +13,8 @@ include $(CLEAR_VARS) LOCAL_SRC_FILES := \ AaptAssets.cpp \ Command.cpp \ + CrunchCache.cpp \ + FileFinder.cpp \ Main.cpp \ Package.cpp \ StringPool.cpp \ diff --git a/Bundle.h b/Bundle.h index 56fe524..539c312 100644 --- a/Bundle.h +++ b/Bundle.h @@ -25,6 +25,7 @@ typedef enum Command { kCommandAdd, kCommandRemove, kCommandPackage, + kCommandCrunch, } Command; /* @@ -42,13 +43,14 @@ public: mManifestPackageNameOverride(NULL), mInstrumentationPackageNameOverride(NULL), mIsOverlayPackage(false), mAutoAddOverlay(false), mGenDependencies(false), - mAssetSourceDir(NULL), mProguardFile(NULL), + mAssetSourceDir(NULL), + mCrunchedOutputDir(NULL), mProguardFile(NULL), mAndroidManifestFile(NULL), mPublicOutputFile(NULL), mRClassDir(NULL), mResourceIntermediatesDir(NULL), mManifestMinSdkVersion(NULL), mMinSdkVersion(NULL), mTargetSdkVersion(NULL), mMaxSdkVersion(NULL), mVersionCode(NULL), mVersionName(NULL), mCustomPackage(NULL), mExtraPackages(NULL), mMaxResVersion(NULL), mDebugMode(false), mNonConstantId(false), mProduct(NULL), - mArgc(0), mArgv(NULL) + mUseCrunchCache(false), mArgc(0), mArgv(NULL) {} ~Bundle(void) {} @@ -106,6 +108,8 @@ public: */ const char* getAssetSourceDir() const { return mAssetSourceDir; } void setAssetSourceDir(const char* dir) { mAssetSourceDir = dir; } + const char* getCrunchedOutputDir() const { return mCrunchedOutputDir; } + void setCrunchedOutputDir(const char* dir) { mCrunchedOutputDir = dir; } const char* getProguardFile() const { return mProguardFile; } void setProguardFile(const char* file) { mProguardFile = file; } const android::Vector& getResourceSourceDirs() const { return mResourceSourceDirs; } @@ -151,6 +155,8 @@ public: void setNonConstantId(bool val) { mNonConstantId = val; } const char* getProduct() const { return mProduct; } void setProduct(const char * val) { mProduct = val; } + void setUseCrunchCache(bool val) { mUseCrunchCache = val; } + bool getUseCrunchCache() { return mUseCrunchCache; } /* * Set and get the file specification. @@ -231,6 +237,7 @@ private: bool mAutoAddOverlay; bool mGenDependencies; const char* mAssetSourceDir; + const char* mCrunchedOutputDir; const char* mProguardFile; const char* mAndroidManifestFile; const char* mPublicOutputFile; @@ -254,6 +261,7 @@ private: bool mDebugMode; bool mNonConstantId; const char* mProduct; + bool mUseCrunchCache; /* file specification */ int mArgc; diff --git a/CacheUpdater.h b/CacheUpdater.h new file mode 100644 index 0000000..0e65589 --- /dev/null +++ b/CacheUpdater.h @@ -0,0 +1,107 @@ +// +// Copyright 2011 The Android Open Source Project +// +// Abstraction of calls to system to make directories and delete files and +// wrapper to image processing. + +#ifndef CACHE_UPDATER_H +#define CACHE_UPDATER_H + +#include +#include +#include +#include +#include "Images.h" + +using namespace android; + +/** CacheUpdater + * This is a pure virtual class that declares abstractions of functions useful + * for managing a cache files. This manager is set up to be used in a + * mirror cache where the source tree is duplicated and filled with processed + * images. This class is abstracted to allow for dependency injection during + * unit testing. + * Usage: + * To update/add a file to the cache, call processImage + * To remove a file from the cache, call deleteFile + */ +class CacheUpdater { +public: + // Make sure all the directories along this path exist + virtual void ensureDirectoriesExist(String8 path) = 0; + + // Delete a file + virtual void deleteFile(String8 path) = 0; + + // Process an image from source out to dest + virtual void processImage(String8 source, String8 dest) = 0; +private: +}; + +/** SystemCacheUpdater + * This is an implementation of the above virtual cache updater specification. + * This implementations hits the filesystem to manage a cache and calls out to + * the PNG crunching in images.h to process images out to its cache components. + */ +class SystemCacheUpdater : public CacheUpdater { +public: + // Constructor to set bundle to pass to preProcessImage + SystemCacheUpdater (Bundle* b) + : bundle(b) { }; + + // Make sure all the directories along this path exist + virtual void ensureDirectoriesExist(String8 path) + { + // Check to see if we're dealing with a fully qualified path + String8 existsPath; + String8 toCreate; + String8 remains; + struct stat s; + + // Check optomistically to see if all directories exist. + // If something in the path doesn't exist, then walk the path backwards + // and find the place to start creating directories forward. + if (stat(path.string(),&s) == -1) { + // Walk backwards to find place to start creating directories + existsPath = path; + do { + // As we remove the end of existsPath add it to + // the string of paths to create. + toCreate = existsPath.getPathLeaf().appendPath(toCreate); + existsPath = existsPath.getPathDir(); + } while (stat(existsPath.string(),&s) == -1); + + // Walk forwards and build directories as we go + do { + // Advance to the next segment of the path + existsPath.appendPath(toCreate.walkPath(&remains)); + toCreate = remains; +#ifdef HAVE_MS_C_RUNTIME + _mkdir(existsPath.string()); +#else + mkdir(existsPath.string(), S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP); +#endif + } while (remains.length() > 0); + } //if + }; + + // Delete a file + virtual void deleteFile(String8 path) + { + if (remove(path.string()) != 0) + fprintf(stderr,"ERROR DELETING %s\n",path.string()); + }; + + // Process an image from source out to dest + virtual void processImage(String8 source, String8 dest) + { + // Make sure we're trying to write to a directory that is extant + ensureDirectoriesExist(dest.getPathDir()); + + preProcessImageToCache(bundle, source, dest); + }; +private: + Bundle* bundle; +}; + +#endif // CACHE_UPDATER_H \ No newline at end of file diff --git a/Command.cpp b/Command.cpp index 8447d74..3bc5fa1 100644 --- a/Command.cpp +++ b/Command.cpp @@ -1504,3 +1504,25 @@ bail: } return retVal; } + +/* + * Do PNG Crunching + * PRECONDITIONS + * -S flag points to a source directory containing drawable* folders + * -C flag points to destination directory. The folder structure in the + * source directory will be mirrored to the destination (cache) directory + * + * POSTCONDITIONS + * Destination directory will be updated to match the PNG files in + * the source directory. + */ +int doCrunch(Bundle* bundle) +{ + fprintf(stdout, "Crunching PNG Files in "); + fprintf(stdout, "source dir: %s\n", bundle->getResourceSourceDirs()[0]); + fprintf(stdout, "To destination dir: %s\n", bundle->getCrunchedOutputDir()); + + updatePreProcessedCache(bundle); + + return NO_ERROR; +} diff --git a/CrunchCache.cpp b/CrunchCache.cpp new file mode 100644 index 0000000..c4cf6bc --- /dev/null +++ b/CrunchCache.cpp @@ -0,0 +1,104 @@ +// +// Copyright 2011 The Android Open Source Project +// +// Implementation file for CrunchCache +// This file defines functions laid out and documented in +// CrunchCache.h + +#include +#include + +#include "DirectoryWalker.h" +#include "FileFinder.h" +#include "CacheUpdater.h" +#include "CrunchCache.h" + +using namespace android; + +CrunchCache::CrunchCache(String8 sourcePath, String8 destPath, FileFinder* ff) + : mSourcePath(sourcePath), mDestPath(destPath), mSourceFiles(0), mDestFiles(0), mFileFinder(ff) +{ + // We initialize the default value to return to 0 so if a file doesn't exist + // then all files are automatically "newer" than it. + + // Set file extensions to look for. Right now just pngs. + mExtensions.push(String8(".png")); + + // Load files into our data members + loadFiles(); +} + +size_t CrunchCache::crunch(CacheUpdater* cu, bool forceOverwrite) +{ + size_t numFilesUpdated = 0; + + // Iterate through the source files and compare to cache. + // After processing a file, remove it from the source files and + // from the dest files. + // We're done when we're out of files in source. + String8 relativePath; + while (mSourceFiles.size() > 0) { + // Get the full path to the source file, then convert to a c-string + // and offset our beginning pointer to the length of the sourcePath + // This efficiently strips the source directory prefix from our path. + // Also, String8 doesn't have a substring method so this is what we've + // got to work with. + const char* rPathPtr = mSourceFiles.keyAt(0).string()+mSourcePath.length(); + // Strip leading slash if present + int offset = 0; + if (rPathPtr[0] == OS_PATH_SEPARATOR) + offset = 1; + relativePath = String8(rPathPtr + offset); + + if (forceOverwrite || needsUpdating(relativePath)) { + cu->processImage(mSourcePath.appendPathCopy(relativePath), + mDestPath.appendPathCopy(relativePath)); + numFilesUpdated++; + // crunchFile(relativePath); + } + // Delete this file from the source files and (if it exists) from the + // dest files. + mSourceFiles.removeItemsAt(0); + mDestFiles.removeItem(mDestPath.appendPathCopy(relativePath)); + } + + // Iterate through what's left of destFiles and delete leftovers + while (mDestFiles.size() > 0) { + cu->deleteFile(mDestFiles.keyAt(0)); + mDestFiles.removeItemsAt(0); + } + + // Update our knowledge of the files cache + // both source and dest should be empty by now. + loadFiles(); + + return numFilesUpdated; +} + +void CrunchCache::loadFiles() +{ + // Clear out our data structures to avoid putting in duplicates + mSourceFiles.clear(); + mDestFiles.clear(); + + // Make a directory walker that points to the system. + DirectoryWalker* dw = new SystemDirectoryWalker(); + + // Load files in the source directory + mFileFinder->findFiles(mSourcePath, mExtensions, mSourceFiles,dw); + + // Load files in the destination directory + mFileFinder->findFiles(mDestPath,mExtensions,mDestFiles,dw); + + delete dw; +} + +bool CrunchCache::needsUpdating(String8 relativePath) const +{ + // Retrieve modification dates for this file entry under the source and + // cache directory trees. The vectors will return a modification date of 0 + // if the file doesn't exist. + time_t sourceDate = mSourceFiles.valueFor(mSourcePath.appendPathCopy(relativePath)); + time_t destDate = mDestFiles.valueFor(mDestPath.appendPathCopy(relativePath)); + return sourceDate > destDate; +} \ No newline at end of file diff --git a/CrunchCache.h b/CrunchCache.h new file mode 100644 index 0000000..be3da5c --- /dev/null +++ b/CrunchCache.h @@ -0,0 +1,102 @@ +// +// Copyright 2011 The Android Open Source Project +// +// Cache manager for pre-processed PNG files. +// Contains code for managing which PNG files get processed +// at build time. +// + +#ifndef CRUNCHCACHE_H +#define CRUNCHCACHE_H + +#include +#include +#include "FileFinder.h" +#include "CacheUpdater.h" + +using namespace android; + +/** CrunchCache + * This class is a cache manager which can pre-process PNG files and store + * them in a mirror-cache. It's capable of doing incremental updates to its + * cache. + * + * Usage: + * Create an instance initialized with the root of the source tree, the + * root location to store the cache files, and an instance of a file finder. + * Then update the cache by calling crunch. + */ +class CrunchCache { +public: + // Constructor + CrunchCache(String8 sourcePath, String8 destPath, FileFinder* ff); + + // Nobody should be calling the default constructor + // So this space is intentionally left blank + + // Default Copy Constructor and Destructor are fine + + /** crunch is the workhorse of this class. + * It goes through all the files found in the sourcePath and compares + * them to the cached versions in the destPath. If the optional + * argument forceOverwrite is set to true, then all source files are + * re-crunched even if they have not been modified recently. Otherwise, + * source files are only crunched when they needUpdating. Afterwards, + * we delete any leftover files in the cache that are no longer present + * in source. + * + * PRECONDITIONS: + * No setup besides construction is needed + * POSTCONDITIONS: + * The cache is updated to fully reflect all changes in source. + * The function then returns the number of files changed in cache + * (counting deletions). + */ + size_t crunch(CacheUpdater* cu, bool forceOverwrite=false); + +private: + /** loadFiles is a wrapper to the FileFinder that places matching + * files into mSourceFiles and mDestFiles. + * + * POSTCONDITIONS + * mDestFiles and mSourceFiles are refreshed to reflect the current + * state of the files in the source and dest directories. + * Any previous contents of mSourceFiles and mDestFiles are cleared. + */ + void loadFiles(); + + /** needsUpdating takes a file path + * and returns true if the file represented by this path is newer in the + * sourceFiles than in the cache (mDestFiles). + * + * PRECONDITIONS: + * mSourceFiles and mDestFiles must be initialized and filled. + * POSTCONDITIONS: + * returns true if and only if source file's modification time + * is greater than the cached file's mod-time. Otherwise returns false. + * + * USAGE: + * Should be used something like the following: + * if (needsUpdating(filePath)) + * // Recrunch sourceFile out to destFile. + * + */ + bool needsUpdating(String8 relativePath) const; + + // DATA MEMBERS ==================================================== + + String8 mSourcePath; + String8 mDestPath; + + Vector mExtensions; + + // Each vector of paths contains one entry per PNG file encountered. + // Each entry consists of a path pointing to that PNG. + DefaultKeyedVector mSourceFiles; + DefaultKeyedVector mDestFiles; + + // Pointer to a FileFinder to use + FileFinder* mFileFinder; +}; + +#endif // CRUNCHCACHE_H diff --git a/DirectoryWalker.h b/DirectoryWalker.h new file mode 100644 index 0000000..88031d0 --- /dev/null +++ b/DirectoryWalker.h @@ -0,0 +1,98 @@ +// +// Copyright 2011 The Android Open Source Project +// +// Defines an abstraction for opening a directory on the filesystem and +// iterating through it. + +#ifndef DIRECTORYWALKER_H +#define DIRECTORYWALKER_H + +#include +#include +#include +#include +#include +#include + +#include + +using namespace android; + +// Directory Walker +// This is an abstraction for walking through a directory and getting files +// and descriptions. + +class DirectoryWalker { +public: + virtual ~DirectoryWalker() {}; + virtual bool openDir(String8 path) = 0; + virtual bool openDir(const char* path) = 0; + // Advance to next directory entry + virtual struct dirent* nextEntry() = 0; + // Get the stats for the current entry + virtual struct stat* entryStats() = 0; + // Clean Up + virtual void closeDir() = 0; + // This class is able to replicate itself on the heap + virtual DirectoryWalker* clone() = 0; + + // DATA MEMBERS + // Current directory entry + struct dirent mEntry; + // Stats for that directory entry + struct stat mStats; + // Base path + String8 mBasePath; +}; + +// System Directory Walker +// This is an implementation of the above abstraction that calls +// real system calls and is fully functional. +// functions are inlined since they're very short and simple + +class SystemDirectoryWalker : public DirectoryWalker { + + // Default constructor, copy constructor, and destructor are fine +public: + virtual bool openDir(String8 path) { + mBasePath = path; + dir = NULL; + dir = opendir(mBasePath.string() ); + + if (dir == NULL) + return false; + + return true; + }; + virtual bool openDir(const char* path) { + String8 p(path); + openDir(p); + return true; + }; + // Advance to next directory entry + virtual struct dirent* nextEntry() { + struct dirent* entryPtr = readdir(dir); + if (entryPtr == NULL) + return NULL; + + mEntry = *entryPtr; + // Get stats + String8 fullPath = mBasePath.appendPathCopy(mEntry.d_name); + stat(fullPath.string(),&mStats); + return &mEntry; + }; + // Get the stats for the current entry + virtual struct stat* entryStats() { + return &mStats; + }; + virtual void closeDir() { + closedir(dir); + }; + virtual DirectoryWalker* clone() { + return new SystemDirectoryWalker(*this); + }; +private: + DIR* dir; +}; + +#endif // DIRECTORYWALKER_H diff --git a/FileFinder.cpp b/FileFinder.cpp new file mode 100644 index 0000000..a1f64d0 --- /dev/null +++ b/FileFinder.cpp @@ -0,0 +1,92 @@ +// +// Copyright 2011 The Android Open Source Project +// + +// File Finder implementation. +// Implementation for the functions declared and documented in FileFinder.h + +#include +#include +#include + + +#include + +#include "DirectoryWalker.h" +#include "FileFinder.h" + +//#define DEBUG + +using android::String8; +using std::cout; +using std::endl; + +bool SystemFileFinder::findFiles(String8 basePath, Vector& extensions, + KeyedVector& fileStore, + DirectoryWalker* dw) +{ + // Scan the directory pointed to by basePath + // check files and recurse into subdirectories. + if (!dw->openDir(basePath)) { + return false; + } +#ifdef DEBUG + cout << "FileFinder looking in " << basePath << endl; +#endif // DEBUG + /* + * Go through all directory entries. Check each file using checkAndAddFile + * and recurse into sub-directories. + */ + struct dirent* entry; + while ((entry = dw->nextEntry()) != NULL) { + String8 entryName(entry->d_name); + if (entry->d_name[0] == '.') // Skip hidden files and directories + continue; + + String8 fullPath = basePath.appendPathCopy(entryName); + // If this entry is a directory we'll recurse into it + if (entry->d_type == DT_DIR) { + DirectoryWalker* copy = dw->clone(); + findFiles(fullPath, extensions, fileStore,copy); + delete copy; + } + + // If this entry is a file, we'll pass it over to checkAndAddFile + if (entry->d_type == DT_REG) { + checkAndAddFile(fullPath,dw->entryStats(),extensions,fileStore); + } + } + + // Clean up + dw->closeDir(); + + return true; +} + +void SystemFileFinder::checkAndAddFile(String8 path, const struct stat* stats, + Vector& extensions, + KeyedVector& fileStore) +{ +#ifdef DEBUG + cout << "Checking file " << path << "..."; +#endif // DEBUG + // Loop over the extensions, checking for a match + bool done = false; + String8 ext(path.getPathExtension()); + ext.toLower(); + for (size_t i = 0; i < extensions.size() && !done; ++i) { + String8 ext2 = extensions[i].getPathExtension(); + ext2.toLower(); + // Compare the extensions. If a match is found, add to storage. + if (ext == ext2) { +#ifdef DEBUG + cout << "Match"; +#endif // DEBUG + done = true; + fileStore.add(path,stats->st_mtime); + } + } +#ifdef DEBUG + cout << endl; +#endif //DEBUG +} \ No newline at end of file diff --git a/FileFinder.h b/FileFinder.h new file mode 100644 index 0000000..3f87aba --- /dev/null +++ b/FileFinder.h @@ -0,0 +1,78 @@ +// +// Copyright 2011 The Android Open Source Project +// + +// File Finder. +// This is a collection of useful functions for finding paths and modification +// times of files that match an extension pattern in a directory tree. +// and finding files in it. + +#ifndef FILEFINDER_H +#define FILEFINDER_H + +#include +#include +#include + +#include "DirectoryWalker.h" + +using namespace android; + +// Abstraction to allow for dependency injection. See MockFileFinder.h +// for the testing implementation. +class FileFinder { +public: + virtual bool findFiles(String8 basePath, Vector& extensions, + KeyedVector& fileStore, + DirectoryWalker* dw) = 0; +}; + +class SystemFileFinder : public FileFinder { +public: + + /* findFiles takes a path, a Vector of extensions, and a destination KeyedVector + * and places path/modification date key/values pointing to + * all files with matching extensions found into the KeyedVector + * PRECONDITIONS + * path is a valid system path + * extensions should include leading "." + * This is not necessary, but the comparison directly + * compares the end of the path string so if the "." + * is excluded there is a small chance you could have + * a false positive match. (For example: extension "png" + * would match a file called "blahblahpng") + * + * POSTCONDITIONS + * fileStore contains (in no guaranteed order) paths to all + * matching files encountered in subdirectories of path + * as keys in the KeyedVector. Each key has the modification time + * of the file as its value. + * + * Calls checkAndAddFile on each file encountered in the directory tree + * Recursively descends into subdirectories. + */ + virtual bool findFiles(String8 basePath, Vector& extensions, + KeyedVector& fileStore, + DirectoryWalker* dw); + +private: + /** + * checkAndAddFile looks at a single file path and stat combo + * to determine whether it is a matching file (by looking at + * the extension) + * + * PRECONDITIONS + * no setup is needed + * + * POSTCONDITIONS + * If the given file has a matching extension then a new entry + * is added to the KeyedVector with the path as the key and the modification + * time as the value. + * + */ + static void checkAndAddFile(String8 path, const struct stat* stats, + Vector& extensions, + KeyedVector& fileStore); + +}; +#endif // FILEFINDER_H diff --git a/Images.cpp b/Images.cpp index 3c471ca..311ceea 100644 --- a/Images.cpp +++ b/Images.cpp @@ -1080,7 +1080,128 @@ bail: return error; } +status_t preProcessImageToCache(Bundle* bundle, String8 source, String8 dest) +{ + png_structp read_ptr = NULL; + png_infop read_info = NULL; + + FILE* fp; + + image_info imageInfo; + + png_structp write_ptr = NULL; + png_infop write_info = NULL; + + status_t error = UNKNOWN_ERROR; + + // Get a file handler to read from + fp = fopen(source.string(),"rb"); + if (fp == NULL) { + fprintf(stderr, "%s ERROR: Unable to open PNG file\n", source.string()); + return error; + } + + // Call libpng to get a struct to read image data into + read_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); + if (!read_ptr) { + fclose(fp); + png_destroy_read_struct(&read_ptr, &read_info,NULL); + return error; + } + + // Call libpng to get a struct to read image info into + read_info = png_create_info_struct(read_ptr); + if (!read_info) { + fclose(fp); + png_destroy_read_struct(&read_ptr, &read_info,NULL); + return error; + } + + // Set a jump point for libpng to long jump back to on error + if (setjmp(png_jmpbuf(read_ptr))) { + fclose(fp); + png_destroy_read_struct(&read_ptr, &read_info,NULL); + return error; + } + + // Set up libpng to read from our file. + png_init_io(read_ptr,fp); + + // Actually read data from the file + read_png(source.string(), read_ptr, read_info, &imageInfo); + // We're done reading so we can clean up + // Find old file size before releasing handle + fseek(fp, 0, SEEK_END); + size_t oldSize = (size_t)ftell(fp); + fclose(fp); + png_destroy_read_struct(&read_ptr, &read_info,NULL); + + // Check to see if we're dealing with a 9-patch + // If we are, process appropriately + if (source.getBasePath().getPathExtension() == ".9") { + if (do_9patch(source.string(), &imageInfo) != NO_ERROR) { + return error; + } + } + + // Call libpng to create a structure to hold the processed image data + // that can be written to disk + write_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); + if (!write_ptr) { + png_destroy_write_struct(&write_ptr, &write_info); + return error; + } + + // Call libpng to create a structure to hold processed image info that can + // be written to disk + write_info = png_create_info_struct(write_ptr); + if (!write_info) { + png_destroy_write_struct(&write_ptr, &write_info); + return error; + } + + // Open up our destination file for writing + fp = fopen(dest.string(), "wb"); + if (!fp) { + fprintf(stderr, "%s ERROR: Unable to open PNG file\n", dest.string()); + png_destroy_write_struct(&write_ptr, &write_info); + return error; + } + + // Set up libpng to write to our file + png_init_io(write_ptr, fp); + + // Set up a jump for libpng to long jump back on on errors + if (setjmp(png_jmpbuf(write_ptr))) { + fclose(fp); + png_destroy_write_struct(&write_ptr, &write_info); + return error; + } + + // Actually write out to the new png + write_png(dest.string(), write_ptr, write_info, imageInfo, + bundle->getGrayscaleTolerance()); + + if (bundle->getVerbose()) { + // Find the size of our new file + FILE* reader = fopen(dest.string(), "rb"); + fseek(reader, 0, SEEK_END); + size_t newSize = (size_t)ftell(reader); + fclose(reader); + + float factor = ((float)newSize)/oldSize; + int percent = (int)(factor*100); + printf(" (processed image to cache entry %s: %d%% size of source)\n", + dest.string(), percent); + } + + //Clean up + fclose(fp); + png_destroy_write_struct(&write_ptr, &write_info); + + return NO_ERROR; +} status_t postProcessImage(const sp& assets, ResourceTable* table, const sp& file) diff --git a/Images.h b/Images.h index 168e22f..4816905 100644 --- a/Images.h +++ b/Images.h @@ -8,11 +8,19 @@ #define IMAGES_H #include "ResourceTable.h" +#include "Bundle.h" + +#include +#include + +using android::String8; status_t preProcessImage(Bundle* bundle, const sp& assets, const sp& file, String8* outNewLeafName); +status_t preProcessImageToCache(Bundle* bundle, String8 source, String8 dest); + status_t postProcessImage(const sp& assets, - ResourceTable* table, const sp& file); + ResourceTable* table, const sp& file); #endif diff --git a/Main.cpp b/Main.cpp index 9887268..5135787 100644 --- a/Main.cpp +++ b/Main.cpp @@ -82,6 +82,9 @@ void usage(void) fprintf(stderr, " %s a[dd] [-v] file.{zip,jar,apk} file1 [file2 ...]\n" " Add specified files to Zip-compatible archive.\n\n", gProgName); + fprintf(stderr, + " %s c[runch] [-v] -S resource-sources ... -C output-folder ...\n" + " Do PNG preprocessing and store the results in output folder.\n\n", gProgName); fprintf(stderr, " %s v[ersion]\n" " Print program version.\n\n", gProgName); @@ -190,6 +193,7 @@ int handleCommand(Bundle* bundle) case kCommandAdd: return doAdd(bundle); case kCommandRemove: return doRemove(bundle); case kCommandPackage: return doPackage(bundle); + case kCommandCrunch: return doCrunch(bundle); default: fprintf(stderr, "%s: requested command not yet supported\n", gProgName); return 1; @@ -227,6 +231,8 @@ int main(int argc, char* const argv[]) bundle.setCommand(kCommandRemove); else if (argv[1][0] == 'p') bundle.setCommand(kCommandPackage); + else if (argv[1][0] == 'c') + bundle.setCommand(kCommandCrunch); else { fprintf(stderr, "ERROR: Unknown command '%s'\n", argv[1]); wantUsage = true; @@ -397,6 +403,17 @@ int main(int argc, char* const argv[]) convertPath(argv[0]); bundle.addResourceSourceDir(argv[0]); break; + case 'C': + argc--; + argv++; + if (!argc) { + fprintf(stderr, "ERROR: No argument supplied for '-C' option\n"); + wantUsage = true; + goto bail; + } + convertPath(argv[0]); + bundle.setCrunchedOutputDir(argv[0]); + break; case '0': argc--; argv++; @@ -523,7 +540,9 @@ int main(int argc, char* const argv[]) bundle.setProduct(argv[0]); } else if (strcmp(cp, "-non-constant-id") == 0) { bundle.setNonConstantId(true); - } else { + } else if (strcmp(cp, "-no-crunch") == 0) { + bundle.setUseCrunchCache(true); + }else { fprintf(stderr, "ERROR: Unknown option '-%s'\n", cp); wantUsage = true; goto bail; diff --git a/Main.h b/Main.h index 9674c5d..d20c601 100644 --- a/Main.h +++ b/Main.h @@ -14,12 +14,21 @@ #include "AaptAssets.h" #include "ZipFile.h" + +/* Benchmarking Flag */ +//#define BENCHMARK 1 + +#if BENCHMARK + #include +#endif /* BENCHMARK */ + extern int doVersion(Bundle* bundle); extern int doList(Bundle* bundle); extern int doDump(Bundle* bundle); extern int doAdd(Bundle* bundle); extern int doRemove(Bundle* bundle); extern int doPackage(Bundle* bundle); +extern int doCrunch(Bundle* bundle); extern int calcPercent(long uncompressedLen, long compressedLen); @@ -27,6 +36,8 @@ extern android::status_t writeAPK(Bundle* bundle, const sp& assets, const android::String8& outputFile); +extern android::status_t updatePreProcessedCache(Bundle* bundle); + extern android::status_t buildResources(Bundle* bundle, const sp& assets); diff --git a/Package.cpp b/Package.cpp index 6e2a25c..62af30e 100644 --- a/Package.cpp +++ b/Package.cpp @@ -50,6 +50,11 @@ ssize_t processJarFiles(Bundle* bundle, ZipFile* zip); status_t writeAPK(Bundle* bundle, const sp& assets, const String8& outputFile) { + #if BENCHMARK + fprintf(stdout, "BENCHMARK: Starting APK Bundling \n"); + long startAPKTime = clock(); + #endif /* BENCHMARK */ + status_t result = NO_ERROR; ZipFile* zip = NULL; int count; @@ -197,6 +202,10 @@ bail: if (result == NO_ERROR && bundle->getVerbose()) printf("Done!\n"); + + #if BENCHMARK + fprintf(stdout, "BENCHMARK: End APK Bundling. Time Elapsed: %f ms \n",(clock() - startAPKTime)/1000.0); + #endif /* BENCHMARK */ return result; } diff --git a/Resource.cpp b/Resource.cpp index 6e86112..5152d3b 100644 --- a/Resource.cpp +++ b/Resource.cpp @@ -10,6 +10,10 @@ #include "ResourceTable.h" #include "Images.h" +#include "CrunchCache.h" +#include "FileFinder.h" +#include "CacheUpdater.h" + #define NOISY(x) // x // ========================================================================== @@ -292,18 +296,19 @@ static status_t makeFileResources(Bundle* bundle, const sp& assets, static status_t preProcessImages(Bundle* bundle, const sp& assets, const sp& set) { - ResourceDirIterator it(set, String8("drawable")); - Vector > newNameFiles; - Vector newNamePaths; bool hasErrors = false; - ssize_t res; - while ((res=it.next()) == NO_ERROR) { - res = preProcessImage(bundle, assets, it.getFile(), NULL); - if (res < NO_ERROR) { - hasErrors = true; + ssize_t res = NO_ERROR; + if (bundle->getUseCrunchCache() == false) { + ResourceDirIterator it(set, String8("drawable")); + Vector > newNameFiles; + Vector newNamePaths; + while ((res=it.next()) == NO_ERROR) { + res = preProcessImage(bundle, assets, it.getFile(), NULL); + if (res < NO_ERROR) { + hasErrors = true; + } } } - return (hasErrors || (res < NO_ERROR)) ? UNKNOWN_ERROR : NO_ERROR; } @@ -753,6 +758,35 @@ status_t massageManifest(Bundle* bundle, sp root) } \ } while (0) +status_t updatePreProcessedCache(Bundle* bundle) +{ + #if BENCHMARK + fprintf(stdout, "BENCHMARK: Starting PNG PreProcessing \n"); + long startPNGTime = clock(); + #endif /* BENCHMARK */ + + String8 source(bundle->getResourceSourceDirs()[0]); + String8 dest(bundle->getCrunchedOutputDir()); + + FileFinder* ff = new SystemFileFinder(); + CrunchCache cc(source,dest,ff); + + CacheUpdater* cu = new SystemCacheUpdater(bundle); + size_t numFiles = cc.crunch(cu); + + if (bundle->getVerbose()) + fprintf(stdout, "Crunched %d PNG files to update cache\n", (int)numFiles); + + delete ff; + delete cu; + + #if BENCHMARK + fprintf(stdout, "BENCHMARK: End PNG PreProcessing. Time Elapsed: %f ms \n" + ,(clock() - startPNGTime)/1000.0); + #endif /* BENCHMARK */ + return 0; +} + status_t buildResources(Bundle* bundle, const sp& assets) { // First, look for a package file to parse. This is required to diff --git a/tests/CrunchCache_test.cpp b/tests/CrunchCache_test.cpp new file mode 100644 index 0000000..20b5022 --- /dev/null +++ b/tests/CrunchCache_test.cpp @@ -0,0 +1,97 @@ +// +// Copyright 2011 The Android Open Source Project +// +#include +#include +#include + +#include "CrunchCache.h" +#include "FileFinder.h" +#include "MockFileFinder.h" +#include "CacheUpdater.h" +#include "MockCacheUpdater.h" + +using namespace android; +using std::cout; +using std::endl; + +void expectEqual(int got, int expected, const char* desc) { + cout << "Checking " << desc << ": "; + cout << "Got " << got << ", expected " << expected << "..."; + cout << ( (got == expected) ? "PASSED" : "FAILED") << endl; + errno += ((got == expected) ? 0 : 1); +} + +int main() { + + errno = 0; + + String8 source("res"); + String8 dest("res2"); + + // Create data for MockFileFinder to feed to the cache + KeyedVector sourceData; + // This shouldn't be updated + sourceData.add(String8("res/drawable/hello.png"),3); + // This should be updated + sourceData.add(String8("res/drawable/world.png"),5); + // This should cause make directory to be called + sourceData.add(String8("res/drawable-cool/hello.png"),3); + + KeyedVector destData; + destData.add(String8("res2/drawable/hello.png"),3); + destData.add(String8("res2/drawable/world.png"),3); + // this should call delete + destData.add(String8("res2/drawable/dead.png"),3); + + // Package up data and create mock file finder + KeyedVector > data; + data.add(source,sourceData); + data.add(dest,destData); + FileFinder* ff = new MockFileFinder(data); + CrunchCache cc(source,dest,ff); + + MockCacheUpdater* mcu = new MockCacheUpdater(); + CacheUpdater* cu(mcu); + + cout << "Running Crunch..."; + int result = cc.crunch(cu); + cout << ((result > 0) ? "PASSED" : "FAILED") << endl; + errno += ((result > 0) ? 0 : 1); + + const int EXPECTED_RESULT = 2; + expectEqual(result, EXPECTED_RESULT, "number of files touched"); + + cout << "Checking calls to deleteFile and processImage:" << endl; + const int EXPECTED_DELETES = 1; + const int EXPECTED_PROCESSED = 2; + // Deletes + expectEqual(mcu->deleteCount, EXPECTED_DELETES, "deleteFile"); + // processImage + expectEqual(mcu->processCount, EXPECTED_PROCESSED, "processImage"); + + const int EXPECTED_OVERWRITES = 3; + result = cc.crunch(cu, true); + expectEqual(result, EXPECTED_OVERWRITES, "number of files touched with overwrite"); + \ + + if (errno == 0) + cout << "ALL TESTS PASSED!" << endl; + else + cout << errno << " TESTS FAILED" << endl; + + delete ff; + delete cu; + + // TESTS BELOW WILL GO AWAY SOON + + String8 source2("ApiDemos/res"); + String8 dest2("ApiDemos/res2"); + + FileFinder* sff = new SystemFileFinder(); + CacheUpdater* scu = new SystemCacheUpdater(); + + CrunchCache scc(source2,dest2,sff); + + scc.crunch(scu); +} \ No newline at end of file diff --git a/tests/FileFinder_test.cpp b/tests/FileFinder_test.cpp new file mode 100644 index 0000000..07bd665 --- /dev/null +++ b/tests/FileFinder_test.cpp @@ -0,0 +1,101 @@ +// +// Copyright 2011 The Android Open Source Project +// +#include +#include +#include +#include +#include +#include + +#include "DirectoryWalker.h" +#include "MockDirectoryWalker.h" +#include "FileFinder.h" + +using namespace android; + +using std::pair; +using std::cout; +using std::endl; + + + +int main() +{ + + cout << "\n\n STARTING FILE FINDER TESTS" << endl; + String8 path("ApiDemos"); + + // Storage to pass to findFiles() + KeyedVector testStorage; + + // Mock Directory Walker initialization. First data, then sdw + Vector< pair > data; + data.push( pair(String8("hello.png"),3) ); + data.push( pair(String8("world.PNG"),3) ); + data.push( pair(String8("foo.pNg"),3) ); + // Neither of these should be found + data.push( pair(String8("hello.jpg"),3) ); + data.push( pair(String8(".hidden.png"),3)); + + DirectoryWalker* sdw = new StringDirectoryWalker(path,data); + + // Extensions to look for + Vector exts; + exts.push(String8(".png")); + + errno = 0; + + // Make sure we get a valid mock directory walker + // Make sure we finish without errors + cout << "Checking DirectoryWalker..."; + assert(sdw != NULL); + cout << "PASSED" << endl; + + // Make sure we finish without errors + cout << "Running findFiles()..."; + bool findStatus = FileFinder::findFiles(path,exts, testStorage, sdw); + assert(findStatus); + cout << "PASSED" << endl; + + const size_t SIZE_EXPECTED = 3; + // Check to make sure we have the right number of things in our storage + cout << "Running size comparison: Size is " << testStorage.size() << ", "; + cout << "Expected " << SIZE_EXPECTED << "..."; + if(testStorage.size() == SIZE_EXPECTED) + cout << "PASSED" << endl; + else { + cout << "FAILED" << endl; + errno++; + } + + // Check to make sure that each of our found items has the right extension + cout << "Checking Returned Extensions..."; + bool extsOkay = true; + String8 wrongExts; + for (size_t i = 0; i < SIZE_EXPECTED; ++i) { + String8 testExt(testStorage.keyAt(i).getPathExtension()); + testExt.toLower(); + if (testExt != ".png") { + wrongExts += testStorage.keyAt(i); + wrongExts += "\n"; + extsOkay = false; + } + } + if (extsOkay) + cout << "PASSED" << endl; + else { + cout << "FAILED" << endl; + cout << "The following extensions didn't check out" << endl << wrongExts; + } + + // Clean up + delete sdw; + + if(errno == 0) { + cout << "ALL TESTS PASSED" << endl; + } else { + cout << errno << " TESTS FAILED" << endl; + } + return errno; +} \ No newline at end of file diff --git a/tests/MockCacheUpdater.h b/tests/MockCacheUpdater.h new file mode 100644 index 0000000..c7f4bd7 --- /dev/null +++ b/tests/MockCacheUpdater.h @@ -0,0 +1,40 @@ +// +// Copyright 2011 The Android Open Source Project +// +#ifndef MOCKCACHEUPDATER_H +#define MOCKCACHEUPDATER_H + +#include +#include "CacheUpdater.h" + +using namespace android; + +class MockCacheUpdater : public CacheUpdater { +public: + + MockCacheUpdater() + : deleteCount(0), processCount(0) { }; + + // Make sure all the directories along this path exist + virtual void ensureDirectoriesExist(String8 path) + { + // Nothing to do + }; + + // Delete a file + virtual void deleteFile(String8 path) { + deleteCount++; + }; + + // Process an image from source out to dest + virtual void processImage(String8 source, String8 dest) { + processCount++; + }; + + // DATA MEMBERS + int deleteCount; + int processCount; +private: +}; + +#endif // MOCKCACHEUPDATER_H \ No newline at end of file diff --git a/tests/MockDirectoryWalker.h b/tests/MockDirectoryWalker.h new file mode 100644 index 0000000..5900cf3 --- /dev/null +++ b/tests/MockDirectoryWalker.h @@ -0,0 +1,85 @@ +// +// Copyright 2011 The Android Open Source Project +// +#ifndef MOCKDIRECTORYWALKER_H +#define MOCKDIRECTORYWALKER_H + +#include +#include +#include +#include "DirectoryWalker.h" + +using namespace android; +using std::pair; + +// String8 Directory Walker +// This is an implementation of the Directory Walker abstraction that is built +// for testing. +// Instead of system calls it queries a private data structure for the directory +// entries. It takes a path and a map of filenames and their modification times. +// functions are inlined since they are short and simple + +class StringDirectoryWalker : public DirectoryWalker { +public: + StringDirectoryWalker(String8& path, Vector< pair >& data) + : mPos(0), mBasePath(path), mData(data) { + //fprintf(stdout,"StringDW built to mimic %s with %d files\n", + // mBasePath.string()); + }; + // Default copy constructor, and destructor are fine + + virtual bool openDir(String8 path) { + // If the user is trying to query the "directory" that this + // walker was initialized with, then return success. Else fail. + return path == mBasePath; + }; + virtual bool openDir(const char* path) { + String8 p(path); + openDir(p); + return true; + }; + // Advance to next entry in the Vector + virtual struct dirent* nextEntry() { + // Advance position and check to see if we're done + if (mPos >= mData.size()) + return NULL; + + // Place data in the entry descriptor. This class only returns files. + mEntry.d_type = DT_REG; + mEntry.d_ino = mPos; + // Copy chars from the string name to the entry name + size_t i = 0; + for (i; i < mData[mPos].first.size(); ++i) + mEntry.d_name[i] = mData[mPos].first[i]; + mEntry.d_name[i] = '\0'; + + // Place data in stats + mStats.st_ino = mPos; + mStats.st_mtime = mData[mPos].second; + + // Get ready to move to the next entry + mPos++; + + return &mEntry; + }; + // Get the stats for the current entry + virtual struct stat* entryStats() { + return &mStats; + }; + // Nothing to do in clean up + virtual void closeDir() { + // Nothing to do + }; + virtual DirectoryWalker* clone() { + return new StringDirectoryWalker(*this); + }; +private: + // Current position in the Vector + size_t mPos; + // Base path + String8 mBasePath; + // Data to simulate a directory full of files. + Vector< pair > mData; +}; + +#endif // MOCKDIRECTORYWALKER_H \ No newline at end of file diff --git a/tests/MockFileFinder.h b/tests/MockFileFinder.h new file mode 100644 index 0000000..da5ea4f --- /dev/null +++ b/tests/MockFileFinder.h @@ -0,0 +1,55 @@ +// +// Copyright 2011 The Android Open Source Project +// + +#ifndef MOCKFILEFINDER_H +#define MOCKFILEFINDER_H + +#include +#include +#include + +#include "DirectoryWalker.h" + +using namespace android; + +class MockFileFinder : public FileFinder { +public: + MockFileFinder (KeyedVector >& files) + : mFiles(files) + { + // Nothing left to do + }; + + /** + * findFiles implementation for the abstraction. + * PRECONDITIONS: + * No checking is done, so there MUST be an entry in mFiles with + * path matching basePath. + * + * POSTCONDITIONS: + * fileStore is filled with a copy of the data in mFiles corresponding + * to the basePath. + */ + + virtual bool findFiles(String8 basePath, Vector& extensions, + KeyedVector& fileStore, + DirectoryWalker* dw) + { + const KeyedVector* payload(&mFiles.valueFor(basePath)); + // Since KeyedVector doesn't implement swap + // (who doesn't use swap??) we loop and add one at a time. + for (size_t i = 0; i < payload->size(); ++i) { + fileStore.add(payload->keyAt(i),payload->valueAt(i)); + } + return true; + } + +private: + // Virtual mapping between "directories" and the "files" contained + // in them + KeyedVector > mFiles; +}; + + +#endif // MOCKFILEFINDER_H \ No newline at end of file -- 2.45.2