wxPython/samples/ide/activegrid/util/fileutils.py

   1 #----------------------------------------------------------------------------
   2 # Name:         fileutils.py
   3 # Purpose:      Active grid miscellaneous utilities
   4 #
   5 # Author:       Jeff Norton
   6 #
   7 # Created:      12/10/04
   8 # CVS-ID:       $Id$
   9 # Copyright:    (c) 2004-2005 ActiveGrid, Inc.
  10 # License:      wxWindows License
  11 #----------------------------------------------------------------------------
  12
  13 import logging
  14 import copy
  15 import os
  16 import shutil
  17 import sys
  18 import zipfile
  19
  20 import activegrid.util.aglogging as aglogging
  21 import activegrid.util.sysutils as sysutils
  22 import activegrid.util.utillang as utillang
  23 from activegrid.util.lang import *
  24
  25 global fileutilsLogger
  26 fileutilsLogger = logging.getLogger("activegrid.util.fileutils")
  27 # FATAL : No logging
  28 # ERROR : No logging
  29 # WARN  : No logging
  30 # INFO  : No logging
  31 # DEBUG : debugging
  32 aglogging.setLevelFatal(fileutilsLogger)
  33 #logging.getLogger().addHandler(logging.StreamHandler(sys.stderr))
  34
  35 def addRef(varname):
  36     return "${%s}" % varname
  37
  38 AG_SYSTEM_VAR_NAMES = [] # all AG System vars, with ${} syntax
  39
  40 AG_SYSTEM_VAR = "AG_SYSTEM"
  41 AG_SYSTEM_VAR_REF = addRef(AG_SYSTEM_VAR)
  42 AG_SYSTEM_VAR_NAMES.append(AG_SYSTEM_VAR_REF)
  43
  44 AG_SYSTEM_STATIC_VAR = "AG_SYSTEM_STATIC"
  45 AG_SYSTEM_STATIC_VAR_REF = addRef(AG_SYSTEM_STATIC_VAR)
  46 AG_SYSTEM_VAR_NAMES.append(AG_SYSTEM_STATIC_VAR_REF)
  47
  48 AG_APP_VAR = "AG_APP"
  49 AG_APP_STATIC_VAR = "AG_APP_STATIC"
  50
  51 # _initAGSystemVars needs to be called to initialize the following two
  52 # containers:
  53 EXPANDED_AG_SYSTEM_VARS = {} # ${varname} -> value (path)
  54 # ${varname}, ordered from longest to shortest path value
  55 AG_SYSTEM_VARS_LENGTH_ORDER = []
  56
  57 def _initAGSystemVars():
  58     if (len(EXPANDED_AG_SYSTEM_VARS) > 0):
  59         return
  60
  61     for v in AG_SYSTEM_VAR_NAMES:
  62         EXPANDED_AG_SYSTEM_VARS[v] = os.path.abspath(expandVars(v))
  63         AG_SYSTEM_VARS_LENGTH_ORDER.append(v)
  64
  65     AG_SYSTEM_VARS_LENGTH_ORDER.sort(_sortByValLength)
  66
  67
  68 def parameterizePathWithAGSystemVar(inpath):
  69     """Returns parameterized path if path starts with a known AG directory. Otherwise returns path as it was passed in."""
  70     _initAGSystemVars()
  71     path = inpath
  72     if not sysutils.isWindows():
  73         # ensure we have forward slashes
  74         path = path.replace("\\", "/")
  75
  76     path = os.path.abspath(path)
  77
  78     for varname in AG_SYSTEM_VARS_LENGTH_ORDER:
  79         varval = EXPANDED_AG_SYSTEM_VARS[varname]
  80         if path.startswith(varval):
  81             return path.replace(varval, varname)
  82
  83     return inpath
  84
  85 def startsWithAgSystemVar(path):
  86     """Returns True if path starts with a known AG system env var, False otherwise."""
  87     for varname in AG_SYSTEM_VAR_NAMES:
  88         if path.startswith(varname):
  89             return True
  90     return False
  91
  92 def _sortByValLength(v1, v2):
  93     return len(EXPANDED_AG_SYSTEM_VARS[v2]) - len(EXPANDED_AG_SYSTEM_VARS[v1])
  94
  95 def makeDirsForFile(filename):
  96     d = os.path.dirname(filename)
  97     if (not os.path.exists(d)):
  98         os.makedirs(d)
  99
 100 def createFile(filename, mode='w'):
 101     f = None
 102     if (not os.path.exists(filename)):
 103         makeDirsForFile(filename)
 104     f = file(filename, mode)
 105     return f
 106
 107 def compareFiles(file1, file2, ignore=None):
 108 ##    result = filecmp.cmp(file1, file2)
 109 ##    if result:
 110 ##        return 0
 111 ##    return -1
 112     file1.seek(0)
 113     file2.seek(0)
 114     while True:
 115         line1 = file1.readline()
 116         line2 = file2.readline()
 117         if (len(line1) == 0):
 118             if (len(line2) == 0):
 119                 return 0
 120             else:
 121                 return -1
 122         elif (len(line2) == 0):
 123             return -1
 124         elif (line1 != line2):
 125             if (ignore != None):
 126                 if (line1.startswith(ignore) or line2.startswith(ignore)):
 127                     continue
 128             line1 = line1.replace(" ", "")
 129             line2 = line2.replace(" ", "")
 130             if (line1 != line2):
 131                 len1 = len(line1)
 132                 len2 = len(line2)
 133                 if ((abs(len1 - len2) == 1) and (len1 > 0) and (len2 > 0)
 134                     and (line1[-1] == "\n") and (line2[-1] == "\n")):
 135                     if (len1 > len2):
 136                         longer = line1
 137                         shorter = line2
 138                     else:
 139                         shorter = line1
 140                         longer = line2
 141                     if ((longer[-2] == "\r") and (longer[:-2] == shorter[:-1])):
 142                         continue
 143                     if ((longer[-2:] == shorter[-2:]) and (longer[-3] == "\r") and (longer[:-3] == shorter[:-2])):
 144                         continue
 145                 return -1
 146
 147 def expandKnownAGVars(value):
 148     return expandVars(value, includeEnv=False)
 149
 150 def expandVars(value, includeEnv=True):
 151     """Syntax: ${myvar,default="default value"}"""
 152     import activegrid.runtime as runtime
 153     sx = value.find("${")
 154     if (sx >= 0):
 155         result = asString(value[:sx])
 156         endx = value.find("}")
 157         if (endx > 1):
 158             defaultValue = None
 159             defsx = value.find(",default=\"")
 160             if ((defsx > sx) and (defsx < endx)):
 161                 varname = value[sx+2:defsx]
 162                 if (value[endx-1] == '"'):
 163                     defaultValue = value[defsx+10:endx-1]
 164             if (defaultValue == None):
 165                 varname = value[sx+2:endx]
 166             if (varname == AG_SYSTEM_VAR):
 167                 varval = runtime.appInfo.getSystemDir()
 168             elif (varname == AG_SYSTEM_STATIC_VAR):
 169                 varval = runtime.appInfo.getSystemStaticDir()
 170             elif (varname == AG_APP_VAR):
 171                 varval = runtime.appInfo.getAppDir()
 172             elif (varname == AG_APP_STATIC_VAR):
 173                 varval = runtime.appInfo.getAppStaticDir()
 174             else:
 175                 if (includeEnv):
 176                     varval = os.getenv(varname)
 177                 else:
 178                     varval = None
 179             if ((varval == None) and (defaultValue != None)):
 180                 varval = defaultValue
 181             if (varval == None):
 182                 result += value[sx:endx+1]
 183             else:
 184                 result += varval
 185             return result + expandVars(value[endx+1:])
 186     return value
 187
 188 def toPHPpath(path, otherdir=None):
 189     return convertSourcePath(path, "php", otherdir=otherdir)
 190
 191 def toPythonpath(path, otherdir=None):
 192     return convertSourcePath(path, "python", otherdir=otherdir)
 193
 194 def toUnixPath(path):
 195     if (path != None and os.sep != '/'):
 196         path = path.replace(os.sep, '/')
 197     return path
 198
 199 def convertSourcePath(path, to, otherdir=None):
 200     fromname = "python"
 201     if (to == "python"):
 202         fromname = "php"
 203     pythonNode = os.sep + fromname + os.sep
 204     ix = path.find(pythonNode)
 205     if (ix < 0):
 206         ix = path.find(fromname) - 1
 207         if ((ix < 0) or (len(path) <= ix+7)
 208             or (path[ix] not in ("\\", "/")) or (path[ix+7]  not in ("\\", "/"))):
 209             raise Exception("Not in a %s source tree.  Cannot create file name for %s." % (fromname, path))
 210         if (otherdir == None):
 211             return path[:ix+1] + to + path[ix+7:]
 212         else:
 213             return otherdir + path[ix+7:]
 214     if (otherdir == None):
 215         return path.replace(pythonNode, os.sep + to + os.sep)
 216     else:
 217         return otherdir + path[ix+7:]
 218
 219
 220 def visit(directory, files, extension, maxLevel=None, level=1):
 221     testdirs = os.listdir(directory)
 222     for thing in testdirs:
 223         fullpath = os.path.join(directory, thing)
 224         if (os.path.isdir(fullpath) and (maxLevel == None or level < maxLevel)):
 225             visit(fullpath, files, extension, maxLevel, level+1)
 226         elif thing.endswith(extension):
 227             fullname = os.path.normpath(os.path.join(directory, thing))
 228             if not fullname in files:
 229                 files.append(fullname)
 230
 231 def listFilesByExtensionInPath(path=[], extension='.lyt', maxLevel=None):
 232     retval = []
 233     for directory in path:
 234         visit(directory, retval, extension, maxLevel)
 235     return retval
 236
 237 def getFileLastModificationTime(fileName):
 238     return os.path.getmtime(fileName)
 239
 240 def findFileLocation(location, fileName):
 241     i = fileName.rfind(os.sep)
 242     if i > 0:
 243         fileName = fileName[:i]
 244     while location[0:2] == '..' and location[2:3] == os.sep:
 245         location = location[3:]
 246         i = fileName.rfind(os.sep)
 247         fileName = fileName[:i]
 248     absPath = fileName + os.sep + location
 249     return absPath
 250
 251 def getAllExistingFiles(files, basepath=None, forceForwardSlashes=False):
 252     """For each file in files, if it exists, adds its absolute path to the rtn list. If file is a dir, calls this function recursively on all child files in the dir.
 253     If basepath is set, and if the file being processed is relative to basedir, adds that relative path to rtn list instead of the abs path.
 254     Is this is Windows, and forceForwardSlashes is True, make sure returned paths only have forward slashes."""
 255     if isinstance(files, basestring):
 256         files = [files]
 257     rtn = []
 258     for file in files:
 259         if os.path.exists(file):
 260             if os.path.isfile(file):
 261                 if basepath and hasAncestorDir(file, basepath):
 262                     rtn.append(getRelativePath(file, basepath))
 263                 else:
 264                     rtn.append(os.path.abspath(str(file)))
 265             elif os.path.isdir(file):
 266                 dircontent = [os.path.join(file, f) for f in os.listdir(file)]
 267                 rtn.extend(getAllExistingFiles(dircontent, basepath))
 268
 269     if forceForwardSlashes and sysutils.isWindows():
 270         newRtn = []
 271         for f in rtn:
 272             newRtn.append(f.replace("\\", "/"))
 273         rtn = newRtn
 274
 275     return rtn
 276
 277 def hasAncestorDir(file, parent):
 278     """Returns true if file has the dir 'parent' as some parent in its path."""
 279     return getRelativePath(file, parent) != None
 280
 281 def getRelativePath(file, basedir):
 282     """Returns relative path from 'basedir' to 'file', assuming 'file' lives beneath 'basedir'. If it doesn't, returns None."""
 283     file = os.path.abspath(file)
 284     parent = os.path.abspath(basedir)
 285
 286     if file == parent:
 287         return None
 288
 289     if file.startswith(parent):
 290         return file[len(parent)+1:]
 291
 292     return None
 293
 294 def isEmptyDir(dir):
 295     if not os.path.isdir(dir):
 296         return False
 297     return len(os.listdir(dir)) == 0
 298
 299 ifDefPy()
 300 def zip(zipfilepath, basedir=None, files=None):
 301     """Zip all files in files and save zip as zipfilepath. If files is None, zip all files in basedir. For all files to be zipped, if they are relative to basedir, include the relative path in the archive."""
 302
 303     if not files and not basedir:
 304         raise AssertionError("Either 'basedir' or 'files' must be set")
 305
 306     if not files:
 307         aglogging.debug(fileutilsLogger,\
 308                         "Looking for files to zip in %s" % basedir)
 309         files = getAllExistingFiles(basedir)
 310     else:
 311         # removes files that don't exist and gets abs for each
 312         files = getAllExistingFiles(files)
 313
 314     if len(files) == 0:
 315         aglogging.debug(fileutilsLogger, "No files to zip, nothing to do")
 316         return
 317
 318     z = zipfile.ZipFile(zipfilepath, mode="w", compression=zipfile.ZIP_DEFLATED)
 319
 320     try:
 321         for file in files:
 322             arcname = None
 323             if basedir:
 324                 arcname = getRelativePath(file, basedir)
 325             if not arcname:
 326                 arcname = file
 327                 aglogging.debug(fileutilsLogger,\
 328                                 "%s: adding %s with arcname %s" %\
 329                                 (zipfilepath, file, arcname))
 330             z.write(file, arcname)
 331     finally:
 332         z.close()
 333 endIfDef()
 334
 335
 336 ifDefPy()
 337 def unzip(zipfilepath, extractdir):
 338     """Unzip zipfilepath into extractdir."""
 339     z = zipfile.ZipFile(zipfilepath, mode="r")
 340     for info in z.infolist():
 341         filename = os.path.join(extractdir, info.filename)
 342         try:
 343             dir = os.path.dirname(filename)
 344             aglogging.debug(fileutilsLogger, "Creating dir %s" % dir)
 345             os.makedirs(dir) # do we have to worry about permissions?
 346         except:
 347             pass
 348         if os.path.isdir(filename):
 349             continue
 350         aglogging.debug(fileutilsLogger,\
 351                        ("Writing arcfile %s to %s" % (info.filename, filename)))
 352         f = open(filename, "w")
 353         f.write(z.read(info.filename))
 354         f.close()
 355 endIfDef()
 356
 357 ifDefPy()
 358 def copyFile(src, dest):
 359     """Copies file src to dest. Creates directories in 'dest' path if necessary."""
 360     destdir = os.path.dirname(dest)
 361     if not os.path.exists(destdir):
 362         os.makedirs(destdir)
 363     shutil.copy(src, dest)
 364 endIfDef()
 365
 366 ifDefPy()
 367 def copyDir(src, dest):
 368     """Copies dir 'src' into dir 'dest'. Creates 'dest' if it does not exist."""
 369     shutil.copytree(src, dest)
 370 endIfDef()
 371
 372 ifDefPy()
 373 def remove(file):
 374     if not os.path.exists(file):
 375         return
 376     if os.path.isfile(file):
 377         os.remove(file)
 378     elif os.path.isdir(file):
 379         shutil.rmtree(file)
 380 endIfDef()
 381
 382 def getUserTempDir():
 383     systemTempDir = utillang.getSystemTempDir()
 384     userName = sysutils.getUserName()
 385     userNameNoSpace = userName.replace('_','__').replace(' ','_')
 386     userTempDir = systemTempDir + os.sep + "activegrid_" + userNameNoSpace
 387     return userTempDir
 388
 389 def createUserTempDir():
 390     userTempDir = getUserTempDir()
 391     if not os.path.exists(userTempDir):
 392         os.makedirs(userTempDir)
 393         os.chmod(userTempDir, 0700)
 394
 395 createUserTempDir()
 396
 397 ifDefPy()
 398 import warnings
 399 warnings.filterwarnings("ignore", message="tmpnam is a potential security risk to your program")
 400 def getTmpFile():
 401     return os.tmpnam()
 402 endIfDef()
 403
 404 ifDefPy()
 405 #@accepts str, dict, str, str, boolean
 406 def replaceToken(infilepath, tokens={}, outfilepath=None, delim="@@",\
 407                  useEnv=False):
 408     """Replaces tokens of form 'delim'<tokenname>'delim' in file at 'infilepath', using values in dict 'tokens'. If 'outfilepath' is set, writes output to 'outfilepath', if not set, overwrites original file. If 'useEnv' is True, adds os.environ to 'tokens'. This makes it possible to define an env var FOO=BLAH, and have @@FOO@@ be replaced with BLAH, without explicitly passing FOO=BLAH in 'tokens'. Note that entries in 'tokens' take precedence over entries in os.environ."""
 409
 410     if useEnv:
 411         for key, val in os.environ.items():
 412             # passed in tokens take precedence
 413             if not tokens.has_key(key):
 414                 tokens[key] = val
 415
 416     f = open(infilepath, "r")
 417     try:
 418         content = f.read()
 419     finally:
 420         if f: f.close()
 421
 422     for token, value in tokens.items():
 423         content = content.replace("%s%s%s" % (delim, token , delim), str(value))
 424
 425     if not outfilepath: outfilepath = infilepath
 426     f = open(outfilepath, "w")
 427     try:
 428         f.write(content)
 429     finally:
 430         if f: f.close()
 431 endIfDef()