Add support for loading config from a clcache.conf file

This is useful and the GCC version of ccache also implements a similar concept.

It might be useful in situations where some default settings could be
specified or when environment is not passed correctly down to the compiler
chain.

This change introduces support for clcache.conf file to allow clcache
to read settings from there.

The precedence is as follows:
 1. If present, Environment variable(s) are used, or
 2. [current_working_dir]\clcache.conf is loaded, or, if not found
 3. [%HOME% or ~\].clcache\clcache.conf is be loaded, or if not found
 4. [%ALLUSERSPROFILE% or C:\Users\].clcache\clcache.conf is loaded.

In each case, once a clcache.conf file is found, no other conf file is
considered and values only from this file are used. It is also loaded
only once (and all its values then cached) - all to avoid unnecessary
performance penalties.

Author: formiaczek

README.asciidoc

@@ -46,8 +46,13 @@ Options
     Sets the maximum size of the cache in bytes.
     The default value is 1073741824 (1 GiB).
 
-Environment Variables
-~~~~~~~~~~~~~~~~~~~~~
+Configuration
+~~~~~~~~~~~~~
+
+Following values are read from Environment variables, and if absent, from either (in that order):
+  - (current-working-dir)\clcache.conf
+  - %HOME%\.clcache\clcache.conf
+  - %ALLUSERSPROFILE%\.clcache\clcache.conf
 
 CLCACHE_DIR::
     If set, points to the directory within which all the cached object files

clcache/__main__.py

@@ -24,7 +24,7 @@
 import sys
 import threading
 from tempfile import TemporaryFile
-from typing import Any, List, Tuple, Iterator
+from typing import Any, Dict, List, Tuple, Iterator
 
 VERSION = "4.1.0-dev"
 
@@ -308,6 +308,50 @@ def getIncludesContentHashForHashes(listOfHashes):
         return HashAlgorithm(','.join(listOfHashes).encode()).hexdigest()
 
 
+class GlobalSettings:
+    """ Implements a common place to obain settings from. """
+    @staticmethod
+    def getValue(settingName, defaultValue=None):
+        value = os.environ.get(settingName, None)
+        if value is None: # compare to None to allow empty values
+            value = GlobalSettings._getFromCache(settingName)
+        return value if value is not None else defaultValue
+
+    # serves as a cache to only read the config file once
+    _cache = {} # type: Dict[str, str]
+
+    @staticmethod
+    def _getFromCache(settingName):
+        if not GlobalSettings._cache:
+            GlobalSettings._readFromFile()
+        return GlobalSettings._cache.get(settingName, None)
+
+    @staticmethod
+    def _readFromFile():
+        GlobalSettings._cache['dummy'] = 'dummy' # so that _readFromFile is only called once
+
+        # prefer config in current directory
+        filename = os.path.join(os.getcwd(), "clcache.conf")
+
+        # ..or in home directory..
+        if not os.path.exists(filename):
+            filename = os.path.join(os.path.expanduser("~"), ".clcache", "clcache.conf")
+
+        # or in "sysconfdir" (%ALLUSERSPROFILE%)
+        if not os.path.exists(filename):
+            dirname = os.environ.get('ALLUSERSPROFILE', None)
+            filename = os.path.join(dirname if dirname else "C:\\Users", ".clcache", "clcache.conf")
+        try:
+            with open(filename) as f:
+                for line in f.readlines():
+                    kv = line.split("=")
+                    if len(kv) != 2 or kv[0].startswith("#"):
+                        continue
+                    GlobalSettings._cache[kv[0].strip()] = kv[1].split("#")[0].strip()
+        except IOError:
+            pass # only ignore file access errors (including not-existing path)
+
+
 class CacheLock:
     """ Implements a lock for the object cache which
     can be used in 'with' statements. """
@@ -359,7 +403,7 @@ def release(self):
 
     @staticmethod
     def forPath(path):
-        timeoutMs = int(os.environ.get('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000))
+        timeoutMs = int(GlobalSettings.getValue('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000))
         lockName = path.replace(':', '-').replace('\\', '-')
         return CacheLock(lockName, timeoutMs)
 
@@ -505,10 +549,8 @@ class CacheFileStrategy:
     def __init__(self, cacheDirectory=None):
         self.dir = cacheDirectory
         if not self.dir:
-            try:
-                self.dir = os.environ["CLCACHE_DIR"]
-            except KeyError:
-                self.dir = os.path.join(os.path.expanduser("~"), "clcache")
+            self.dir = GlobalSettings.getValue("CLCACHE_DIR",
+                                               os.path.join(os.path.expanduser("~"), "clcache"))
 
         manifestsRootDir = os.path.join(self.dir, "manifests")
         ensureDirectoryExists(manifestsRootDir)
@@ -593,9 +635,10 @@ def clean(self, stats, maximumSize):
 
 class Cache:
     def __init__(self, cacheDirectory=None):
-        if os.environ.get("CLCACHE_MEMCACHED"):
+        memcached = GlobalSettings.getValue("CLCACHE_MEMCACHED")
+        if memcached and memcached not in ['0', 'false', 'False']:
             from .storage import CacheFileWithMemcacheFallbackStrategy
-            self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"),
+            self.strategy = CacheFileWithMemcacheFallbackStrategy(memcached,
                                                                   cacheDirectory=cacheDirectory)
         else:
             self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory)
@@ -900,7 +943,8 @@ def getCompilerHash(compilerBinary):
 
 
 def getFileHashes(filePaths):
-    if 'CLCACHE_SERVER' in os.environ:
+    server = GlobalSettings.getValue('CLCACHE_SERVER')
+    if server and server not in ['0', 'false', 'False']:
         pipeName = r'\\.\pipe\clcache_srv'
         while True:
             try:
@@ -939,7 +983,7 @@ def getStringHash(dataString):
 
 
 def expandBasedirPlaceholder(path):
-    baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR'))
+    baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR'))
     if path.startswith(BASEDIR_REPLACEMENT):
         if not baseDir:
             raise LogicException('No CLCACHE_BASEDIR set, but found relative path ' + path)
@@ -949,7 +993,7 @@ def expandBasedirPlaceholder(path):
 
 
 def collapseBasedirToPlaceholder(path):
-    baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR'))
+    baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR'))
     if baseDir is None:
         return path
     else:
@@ -971,8 +1015,8 @@ def ensureDirectoryExists(path):
 
 def copyOrLink(srcFilePath, dstFilePath):
     ensureDirectoryExists(os.path.dirname(os.path.abspath(dstFilePath)))
-
-    if "CLCACHE_HARDLINK" in os.environ:
+    hardlink = GlobalSettings.getValue("CLCACHE_HARDLINK")
+    if hardlink and hardlink not in ['0', 'false', 'False']:
         ret = windll.kernel32.CreateHardLinkW(str(dstFilePath), str(srcFilePath), None)
         if ret != 0:
             # Touch the time stamp of the new link so that the build system
@@ -998,11 +1042,10 @@ def myExecutablePath():
 
 
 def findCompilerBinary():
-    if "CLCACHE_CL" in os.environ:
-        path = os.environ["CLCACHE_CL"]
+    path = GlobalSettings.getValue("CLCACHE_CL")
+    if path:
         if os.path.basename(path) == path:
             path = which(path)
-
         return path if os.path.exists(path) else None
 
     frozenByPy2Exe = hasattr(sys, "frozen")
@@ -1020,7 +1063,8 @@ def findCompilerBinary():
 
 
 def printTraceStatement(msg: str) -> None:
-    if "CLCACHE_LOG" in os.environ:
+    clcachelog = GlobalSettings.getValue("CLCACHE_LOG")
+    if clcachelog and clcachelog not in ['0', 'false', 'False']:
         scriptDir = os.path.realpath(os.path.dirname(sys.argv[0]))
         with OUTPUT_LOCK:
             print(os.path.join(scriptDir, "clcache.py") + " " + msg)
@@ -1570,8 +1614,8 @@ def main():
 
     printTraceStatement("Found real compiler binary at '{0!s}'".format(compiler))
     printTraceStatement("Arguments we care about: '{}'".format(sys.argv))
-
-    if "CLCACHE_DISABLE" in os.environ:
+    enabled = GlobalSettings.getValue("CLCACHE_DISABLE")
+    if enabled and enabled not in ['0', 'false', 'False']:
         return invokeRealCompiler(compiler, sys.argv[1:])[0]
     try:
         return processCompileRequest(cache, compiler, sys.argv)
@@ -1670,8 +1714,8 @@ def processSingleSource(compiler, cmdLine, sourceFile, objectFile, environment):
     try:
         assert objectFile is not None
         cache = Cache()
-
-        if 'CLCACHE_NODIRECT' in os.environ:
+        nodirect = GlobalSettings.getValue('CLCACHE_NODIRECT')
+        if nodirect and nodirect not in ['0', 'false', 'False']:
             return processNoDirect(cache, objectFile, compiler, cmdLine, environment)
         else:
             return processDirect(cache, objectFile, compiler, cmdLine, sourceFile)
@@ -1770,7 +1814,8 @@ def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, ex
 
 
 if __name__ == '__main__':
-    if 'CLCACHE_PROFILE' in os.environ:
+    profilingEnabled = GlobalSettings.getValue('CLCACHE_PROFILE')
+    if profilingEnabled and profilingEnabled not in ['0', 'false', 'False']:
         INVOCATION_HASH = getStringHash(','.join(sys.argv))
         cProfile.run('main()', filename='clcache-{}.prof'.format(INVOCATION_HASH))
     else:

tests/test_integration.py

@@ -47,6 +47,28 @@ def cd(targetDirectory):
         os.chdir(oldDirectory)
 
 
+def executeStatsCommand(customEnv=None):
+    cmd = CLCACHE_CMD +  ["-s"]
+    if customEnv:
+        out = subprocess.check_output(cmd, env=customEnv)
+    else:
+        out = subprocess.check_output(cmd)
+    return extractStatsOutput(out.decode("ascii").strip())
+
+
+def extractStatsOutput(outputLines):
+    stats = dict()
+    print(outputLines)
+    for line in outputLines.splitlines():
+        kv = line.split(":", 1)
+        if len(kv) != 2 or not kv[1]:
+            continue
+        stats[kv[0].strip()] = kv[1].strip()
+    # special case to avoid duplication: Update 'Disc cache at X:\\blah\\ccache' => 'X:\\blah\\ccache'
+    stats["current cache dir"] = stats["current cache dir"].split("cache at")[1].strip()
+    return stats
+
+
 class TestCommandLineArguments(unittest.TestCase):
     def testValidMaxSize(self):
         with tempfile.TemporaryDirectory() as tempDir:
@@ -74,6 +96,87 @@ def testPrintStatistics(self):
                 0,
                 "Command must be able to print statistics")
 
+
+class TestGlobalSettings(unittest.TestCase):
+    def testSettingsDefault(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            customEnv = dict(os.environ, HOME=tempDir)
+            stats = executeStatsCommand(customEnv)
+            print(stats)
+            self.assertEqual(stats["current cache dir"], os.path.join(tempDir, "clcache"))
+
+    def testSettingsEnvironmentVariables(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+            stats = executeStatsCommand(customEnv)
+            print(stats)
+            self.assertEqual(stats["current cache dir"], os.path.join(tempDir))
+
+    def testSettingsLocalConfigFile(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            with cd(tempDir):
+                confFileName = os.path.join(tempDir, "clcache.conf")
+                clcacheDir = os.path.join(tempDir, "clcache")
+                self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+                stats = executeStatsCommand()
+                self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testConfigFileInHomeDir(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+            customEnv = dict(os.environ, HOME=tempDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testHomeDirOverridenByEnvironment(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR="this should be ignored")
+            customEnv = dict(os.environ, HOME=tempDir, CLCACHE_DIR=clcacheDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testSettingsConfigFileInProfiles(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+            customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testConfProfilesOverridenByEnvironment(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR="should be ignored")
+            customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir, CLCACHE_DIR=clcacheDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testProfilesOverridenByHomeDir(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            clcacheDir = os.path.join(tempDir, "clcache")
+            homeDir = os.path.join(tempDir, "home")
+            self._createConfFile(os.path.join(homeDir, ".clcache", "clcache.conf"), CLCACHE_DIR=clcacheDir)
+            profilesDir = os.path.join(tempDir, "allusersprofile")
+            self._createConfFile(os.path.join(profilesDir, ".clcache", "clcache.conf"), CLCACHE_DIR="ignored")
+            customEnv = dict(os.environ, HOME=homeDir, ALLUSERSPROFILE=profilesDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def _createConfFile(self, filename, **settings):
+        dirname = os.path.dirname(filename)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(filename, "w") as f:
+            for k, v in settings.items():
+                f.write("{0} = {1}\n\r".format(k, v))
+
+
 class TestDistutils(unittest.TestCase):
     @pytest.mark.skipif(not MONKEY_LOADED, reason="Monkeypatch not loaded")
     @pytest.mark.skipif(CLCACHE_MEMCACHED, reason="Fails with memcached")