From: Tobias Ulmer <tu@emlix.com>
Date: Fri, 2 Sep 2016 11:10:58 +0000 (+0200)
Subject: cache: rework the caching strategy
X-Git-Tag: e2factory-2.3.15rc1~108
X-Git-Url: https://git.e2factory.org/?a=commitdiff_plain;h=636f41579770bb5dd8d51575fa12911f4f952f8f;p=e2factory.git

cache: rework the caching strategy

Introduce new functions to fetch and push files in a consistent way.

To query server flags we now have:

- cache.cache_enabled()
- cache.islocal_enabled()
- cache.writeback_enabled()

All take the optional flags argument, allowing for overrides when
necessary.

The flags table is now checked for unknown flags, throwing a runtime
error. Unused flags have been removed.

Removed the function to cache files explicitly, return the file path if
in cache, and removed some dead code.
If the cache is enabled, calls to any function requesting path or a
copy of the requested file will cache automatically. This behaviour can
now be suppressed with the flags argument.

Introduced a function to check for the existence of a file on the remote
side without downloading:
transport.file_exists()

Added new function
- cache.fetch_file_path()
that returns the requested files path to either the cache, local
file system or a temporary location.

The code base was adjusted to work without any caching if desired,
although this is not a recommended mode of operation due to speed and
temporary storage requirements.

Signed-off-by: Tobias Ulmer <tu@emlix.com>
---

diff --git a/generic/cache.lua b/generic/cache.lua
index 7f6978f..7899657 100644
--- a/generic/cache.lua
+++ b/generic/cache.lua
@@ -85,7 +85,6 @@ function cache.servers(c)
 end
 
 local function assertFlags(flags)
-    flags = flags or {}
     local known = {
         cachable = "boolean",
         cache = "boolean",
@@ -255,23 +254,48 @@ end
 --- check if a cache is enabled
 -- @param c a cache table
 -- @param server the server name
+-- @param flags optional flags table
 -- @return bool
 -- @return an error object on failure
-function cache.cache_enabled(c, server)
+function cache.cache_enabled(c, server, flags)
+    assertIsTable(c)
+    assertIsStringN(server)
+    flags = flags or {}
+    assertFlags(flags)
+
     local ce, re = cache.ce_by_server(c, server)
     if not ce then
         return false, re
     end
-    return ce.flags.cache
+
+    if flags.cache == true then
+        return true
+    elseif ce.flags.cache == true and flags.cache ~= false then
+        return true
+    end
+
+    return false
 end
 
---- check if a file is available in the cache
+--- Check if a file is available in the cache
 -- @param c a cache table
 -- @param server the server name
 -- @param location location relative to the server url
--- @return bool
--- @return an error object on failure
-function cache.file_in_cache(c, server, location)
+-- @param flags optional flags table
+-- @return True if file is in cache, false otherwise
+-- @return Error object on failure
+-- @return Absolute filepath if it is in cache
+local function file_in_cache(c, server, location, flags)
+    assertIsTable(c)
+    assertIsStringN(server)
+    assertIsStringN(location)
+    flags = flags or {}
+    assertFlags(flags)
+
+    if not cache.cache_enabled(c, server, flags) then
+        return false
+    end
+
     local ce, re = cache.ce_by_server(c, server)
     if not ce then
         return false, re
@@ -280,11 +304,113 @@ function cache.file_in_cache(c, server, location)
     if not ceurl then
         return false, re
     end
-    local cf = string.format("/%s/%s", ceurl.path, location)
-    local rc, re = e2lib.isfile(cf)
+    local cf = e2lib.join("/", ceurl.path, location)
+    local rc, re = e2lib.stat(cf)
     if not rc then
         return false
     end
+    return true, nil, cf
+end
+
+function cache.islocal_enabled(c, server, flags)
+    assertIsTable(c)
+    assertIsStringN(server)
+    flags = flags or {}
+    assertFlags(flags)
+
+    local rc, re, ce
+
+    ce, re = cache.ce_by_server(c, server)
+    if not ce then
+        return false, re
+    end
+
+    if flags.islocal == true then
+        return true
+    elseif ce.flags.islocal == true and flags.islocal ~= false then
+        return true
+    end
+
+    return false
+end
+
+
+local function file_is_local(c, server, location, flags)
+    assertIsTable(c)
+    assertIsStringN(server)
+    assertIsStringN(location)
+    flags = flags or {}
+    assertFlags(flags)
+
+    local rc, re
+    local ce, u, filepath
+
+    if not cache.islocal_enabled(c, server, flags) then
+        return false
+    end
+
+    ce, re = cache.ce_by_server(c, server)
+    if not ce then
+        return false, re
+    end
+
+    u, re = url.parse(ce.remote_url)
+    if not u then
+        return false, re
+    end
+
+    if u.transport == "file" then
+        filepath = e2lib.join("/", u.path, location)
+        rc, re = e2lib.stat(filepath)
+        if not rc then
+            return false
+        end
+
+        return true, nil, filepath
+    end
+
+    return false
+end
+
+--- cache a file
+-- @param c the cache data structure
+-- @param server the server to fetch the file from
+-- @param location the location on the server
+-- @param flags
+-- @return bool
+-- @return an error object on failure
+local function cache_file(c, server, location, flags)
+    e2lib.logf(4, "called cache_file from here: %s", debug.traceback("", 2))
+    local e = err.new("caching file failed: %s:%s", server, location)
+    local rc, re
+    local ce, re = cache.ce_by_server(c, server)
+    if not ce then
+        return false, e:cat(re)
+    end
+    assertFlags(flags)
+
+    if not cache.cache_enabled(c, server, flags) then
+        return false, e:append("caching is disabled")
+    end
+
+    local ceurl, re = url.parse(ce.cache_url)
+    if not ceurl then
+        return false, e:cat(re)
+    end
+    local avail, re = file_in_cache(c, server, location)
+    if re then
+        return false, e:cat(re)
+    end
+    if not avail then
+        local destdir = e2lib.join("/", ceurl.path, e2lib.dirname(location))
+        -- fetch the file to the cache
+        rc, re = transport.fetch_file(ce.remote_url, location,
+            destdir, e2lib.basename(location))
+        if not rc then
+            return false, e:cat(re)
+        end
+    end
+
     return true
 end
 
@@ -298,24 +424,26 @@ end
 -- @return bool
 -- @return an error object on failure
 function cache.fetch_file(c, server, location, destdir, destname, flags)
+    assertIsTable(c)
+    assertIsStringN(server)
+    assertIsStringN(location)
+    assertIsStringN(destdir)
+    destname = destname or e2lib.basename(location)
+    assertIsStringN(destname)
+    flags = flags or {}
+    assertFlags(flags)
+
     local rc, re
     local e = err.new("cache: fetching file failed")
     local ce, re = cache.ce_by_server(c, server)
     if not ce then
         return false, e:cat(re)
     end
-    if not destname then
-        destname = e2lib.basename(location)
-    end
-    if not flags then
-        flags = {}
-    end
-    assertFlags(flags)
-    -- fetch the file
-    if ce.flags.cache and flags.cache ~= false then
+
+    if cache.cache_enabled(c, server, flags) then
         -- cache is enabled:
         -- fetch from source to cache and from cache to destination
-        rc, re = cache.cache_file(c, server, location, flags)
+        rc, re = cache_file(c, server, location, flags)
         if not rc then
             return false, e:cat(re)
         end
@@ -335,6 +463,122 @@ function cache.fetch_file(c, server, location, destdir, destname, flags)
     return true
 end
 
+--- Return file path to requested file. The pathname may either point to
+-- the cache, local server or to a temporary file. Do not modify the file!
+-- If the filepath points to a temporary copy, the third return value
+-- is true.
+-- @param c a cache table
+-- @param server the server name
+-- @param location location relative to the server url
+-- @param flags table of flags (optional)
+-- @return filepath to requested file or false on error
+-- @return error object on failure
+-- @return true if temporary file, nil otherwise
+function cache.fetch_file_path(c, server, location, flags)
+    assertIsTable(c)
+    assertIsStringN(server)
+    assertIsStringN(location)
+    flags = flags or {}
+    assertFlags(flags)
+
+    local rc, re, e
+    local ce, filepath
+
+    e = err.new("fetching file to provide file path failed")
+    ce, re = cache.ce_by_server(c, server)
+    if not ce then
+        return false, e:cat(re)
+    end
+
+    -- If you enabled the cache, you probably prefer files from there
+    if cache.cache_enabled(c, server, flags) then
+        rc, re = cache_file(c, server, location, flags)
+        if not rc then
+            return false, e:cat(re)
+        end
+
+        rc, re, filepath = file_in_cache(c, server, location, flags)
+        if not rc and re then
+            return false, e:cat(re)
+        end
+
+        assertTrue(rc)
+        assertIsNil(re)
+        assertIsStringN(filepath)
+        return filepath
+    end
+
+    -- Second choice, the local filesystem
+    rc, re, filepath = file_is_local(c, server, location, flags)
+    if not rc and re then
+        return false, e:cat(re)
+    elseif rc then
+        assertIsNil(re)
+        assertIsStringN(filepath)
+        return filepath
+    end
+
+    -- OK, we're getting a copy for you.
+    filepath, re = e2lib.mktempdir()
+    if not filepath then
+        return false, e:cat(re)
+    end
+    -- preserve the original name for file suffix info etc.
+    filepath = e2lib.join(filepath, e2lib.basename(location))
+
+    rc, re = cache.fetch_file(c, server, location, e2lib.dirname(filepath),
+        e2lib.basename(filepath), flags)
+    if not rc then
+        return false, e:cat(re)
+    end
+
+    return filepath, nil, true
+end
+
+--- Check whether file exists in cache, locally or remote. Please note
+-- error detection doesn't work in all cases, eg. it's difficult to
+-- determine whether a file doesn't exists or there's just a connection problem.
+-- @param c cache
+-- @param server server name
+-- @param location path on server
+-- @param flags table of flags (optional)
+-- @return true if file exists, false otherwise
+-- @return error object if there was an detectable error
+function cache.file_exists(c, server, location, flags)
+    assertIsTable(c)
+    assertIsStringN(server)
+    assertIsStringN(location)
+    flags = flags or {}
+    assertFlags(flags)
+
+    local rc, re, e
+    local ce
+
+    e = err.new("cache: file_exists failed")
+    ce, re = cache.ce_by_server(c, server)
+    if not ce then
+        return false, e:cat(re)
+    end
+
+    if cache.cache_enabled(c, server, flags) then
+        rc, re = file_in_cache(c, server, location, flags)
+        if re then
+            return false, e:cat(re)
+        end
+
+        if rc then
+            return true
+        end
+    end
+
+    rc, re = transport.file_exists(ce.remote_url, location)
+    if re then
+        return false, e:cat(re)
+    end
+
+    return rc
+end
+
 --- push a file to a server: cache and writeback
 -- @param c a cache table
 -- @param sourcefile where to store the file locally
@@ -344,14 +588,17 @@ end
 -- @return bool
 -- @return an error object on failure
 function cache.push_file(c, sourcefile, server, location, flags)
+    flags = flags or {}
+    assertFlags(flags)
+
     local rc, re
     local e = err.new("error pushing file to cache/server")
     local ce, re = cache.ce_by_server(c, server)
     if not ce then
         return false, e:cat(re)
     end
-    assertFlags(flags)
-    if ce.flags.cache and flags.cache ~= false then
+
+    if cache.cache_enabled(c, server, flags) then
         -- cache is enabled:
         -- push the file from source to cache and from cache to
         -- destination
@@ -384,10 +631,10 @@ end
 -- @param server Server name.
 -- @param flags Flags table.
 -- @return Boolean state of writeback.
-function cache.writeback_state(c, server, flags)
+function cache.writeback_enabled(c, server, flags)
     assert(type(c) == "table", "invalid cache")
     assert(type(server) == "string" and server ~= "", "invalid server")
-    assert(type(flags) == "table", "invalid flags")
+    flags = flags or {}
     assertFlags(flags)
 
     local ce, re
@@ -399,7 +646,7 @@ function cache.writeback_state(c, server, flags)
 
     if flags.writeback == false then
         return false
-    elseif ce.flags.writeback == false and flags.writeback == nil then
+    elseif ce.flags.writeback == false and flags.writeback ~= true then
         return false
     end
 
@@ -428,7 +675,7 @@ function cache.writeback(c, server, location, flags)
         return false, e:cat(re)
     end
 
-    if cache.writeback_state(c, server, flags) == false then
+    if cache.writeback_enabled(c, server, flags) == false then
         return true
     end
 
@@ -441,78 +688,6 @@ function cache.writeback(c, server, location, flags)
     return true
 end
 
---- cache a file
--- @param c the cache data structure
--- @param server the server to fetch the file from
--- @param location the location on the server
--- @param flags
--- @return bool
--- @return an error object on failure
-function cache.cache_file(c, server, location, flags)
-    local e = err.new("caching file failed: %s:%s", server, location)
-    local rc, re
-    local ce, re = cache.ce_by_server(c, server)
-    if not ce then
-        return false, e:cat(re)
-    end
-    assertFlags(flags)
-    if not ce.flags.cache then
-        return true
-    end
-    local ceurl, re = url.parse(ce.cache_url)
-    if not ceurl then
-        return false, e:cat(re)
-    end
-    local avail, re = cache.file_in_cache(c, server, location)
-    if re then
-        return false, e:cat(re)
-    end
-    if not avail then
-        local destdir = e2lib.join("/", ceurl.path, e2lib.dirname(location))
-        -- fetch the file to the cache
-        rc, re = transport.fetch_file(ce.remote_url, location, destdir, nil)
-        if not rc then
-            return false, e:cat(re)
-        end
-    end
-    return true
-end
-
---- get path to a cached file or a file on a local server
--- The user must cache the file first using cache.cache_file()
--- @param c the cache data structure
--- @param server the server where the file is located
--- @param location the location on the server
--- @return string the path to the cached file, false on error
--- @return an error object on failure
-function cache.file_path(c, server, location)
-    local rc, re, e
-    local ce, path
-
-    e = err.new("providing path to file not possible (remote server, no cache?)")
-
-    ce, re = cache.ce_by_server(c, server)
-    if not ce then
-        return false, e:cat(re)
-    end
-
-    if ce.flags.cache then
-        path, re = transport.file_path(ce.cache_url, location)
-        if not path then
-            return false, e:cat(re)
-        end
-        return path
-    end
-
-    -- try if the transport delivers a path directly (works for file://)
-    path, re = transport.file_path(ce.remote_url, location)
-    if not path then
-        e:append("Try to enable caching for this server.")
-        return false, e:cat(re)
-    end
-    return path
-end
-
 --- enable/disable writeback for a server
 -- @param c the cache data structure
 -- @param server the server where the file is located
diff --git a/generic/transport.lua b/generic/transport.lua
index 4f99d62..01788ae 100644
--- a/generic/transport.lua
+++ b/generic/transport.lua
@@ -270,6 +270,76 @@ function transport.fetch_file(surl, location, destdir, destname)
     return true
 end
 
+--- Check if remote file exists without downloading.
+-- Note that some transports make it difficult to determine errors or can
+-- generate false positives. Don't rely too much on this function.
+-- @param surl Server URL (string)
+-- @param location Path to file relative to server URL.
+-- @return True if file exists, false if it does not exists or an error occurred.
+-- @return Error object on failure.
+function transport.file_exists(surl, location)
+    assertIsStringN(surl)
+    assertIsStringN(location)
+
+    local rc, re, e
+    local u, filename
+
+    e = err.new("checking if file at %s/%s file_exists failed", surl, location)
+    u, re = url.parse(surl)
+    if not u then
+        return false, e:cat(re)
+    end
+
+    if u.transport == "file" then
+        filename = e2lib.join("/", u.path, location)
+        rc, re = e2lib.exists(filename, false)
+        if rc then
+            return true
+        end
+        return false
+    elseif u.transport == "rsync+ssh" then
+        filename = e2lib.join("/", u.path, location)
+        filename = rsync_quote_remote(u.user, u.servername, filename)
+
+        rc, re = rsync_ssh({ "-n" }, filename, "/")
+        -- can't check for real errors easily
+        return rc
+    elseif u.transport == "scp" or u.transport == "ssh" then
+        filename = e2lib.join("/", u.path, location)
+
+        local test_e, test_not_e
+
+        test_e, re = e2lib.ssh_remote_cmd(u, { "test", "-e", filename})
+        test_not_e, re = e2lib.ssh_remote_cmd(u, { "test", "!", "-e", filename})
+
+        if not test_e and not test_not_e then
+            -- both false, we have a connection issue
+            return false, e:cat(re)
+        elseif test_e and not test_not_e then
+            return true
+        end
+        assert(test_e ~= test_not_e, "schroedingers file?")
+        return false
+    elseif u.transport == "http" or u.transport == "https" then
+        local curl_argv = {}
+
+        filename =  string.format("%s/%s",  u.url, location)
+        table.insert(curl_argv, "-o")
+        table.insert(curl_argv, "/dev/null")
+        table.insert(curl_argv, "--silent")
+        table.insert(curl_argv, "--head")
+        table.insert(curl_argv, "--fail")
+        table.insert(curl_argv, filename)
+
+        rc, re = e2lib.curl(curl_argv)
+        -- can't check for real errors easily
+        return rc
+    end
+
+    e:append("file_exists() not implemented for %s://", u.transport)
+    return false, e
+end
+
 --- push a file to a server
 -- @param sourcefile local file
 -- @param durl url to the destination server
@@ -396,24 +466,6 @@ function transport.push_file(sourcefile, durl, location, push_permissions, try_h
     return true, nil
 end
 
---- Get file path to the specified location, if the file is locally accessable.
--- It's the responsibilty of the caller to check whether the file exists.
--- @param surl Url to the server.
--- @param location Location relative to the server url.
--- @return File path on success, false on error.
--- @return Error string on failure.
-function transport.file_path(surl, location)
-    local e = err.new("can't get path to file")
-    local u, re = url.parse(surl)
-    if not u then
-        return false, e:cat(re)
-    end
-    if u.transport ~= "file" then
-        return false, e:append("transport does not support file_path()")
-    end
-    return string.format("/%s/%s", u.path, location)
-end
-
 return strict.lock(transport)
 
 -- vim:sw=4:sts=4:et:
diff --git a/local/e2-fetch-sources.lua b/local/e2-fetch-sources.lua
index ee52711..10279c8 100644
--- a/local/e2-fetch-sources.lua
+++ b/local/e2-fetch-sources.lua
@@ -117,10 +117,12 @@ local function e2_fetch_source(arg)
         for _,g in ipairs(chroot.groups_sorted) do
             grp = chroot.groups_byname[g]
             for file in grp:file_iter() do
-                rc, re = cache.cache_file(info.cache, file.server,
-                    file.location, {})
-                if not rc then
-                    return false, re
+                if cache.cache_enabled(info.cache, file.server) then
+                    rc, re = cache.fetch_file_path(info.cache, file.server,
+                        file.location)
+                    if not rc then
+                        return false, re
+                    end
                 end
             end
         end
diff --git a/local/e2build.lua b/local/e2build.lua
index 0109aae..3609fbd 100644
--- a/local/e2build.lua
+++ b/local/e2build.lua
@@ -225,28 +225,28 @@ function e2build.build_process_class:_result_available(res, return_flags)
         return_flags.stop = false
         return true
     end
+
     local server, location =
         res:build_mode().storage(info.project_location, project.release_id())
     local dep_set = res:build_mode().dep_set(buildid)
+    local result_location = e2lib.join(location, res:get_name(),
+        dep_set, "result.tar")
 
-    -- cache the result
-    local result_location = e2lib.join(location, res:get_name(), dep_set, "result.tar")
-    local cache_flags = {}
-    rc, re = cache.cache_file(info.cache, server, result_location, cache_flags)
-    if not rc then
-        e2lib.log(3, "caching result failed")
-        -- ignore
+    rc, re = cache.file_exists(info.cache, server, result_location)
+    if re then
+        return false, e:cat(re)
     end
-    local path, re = cache.file_path(info.cache, server, result_location)
-    rc = e2lib.isfile(path)
+
     if not rc then
         -- result is not available. Build.
         return_flags.message = e2lib.align(columns,
             0, string.format("building %-20s", res:get_name()),
             columns, string.format("[%s]", sbid))
         return_flags.stop = false
+
         return true
     end
+
     e2lib.log(3, "result is available locally")
     --[[
     rc, re = update_result_timestamp(info, server, location)
@@ -256,11 +256,12 @@ function e2build.build_process_class:_result_available(res, return_flags)
     -- and push the updated metadata to the server again, if the result
     -- exists on the server.
     ]]
+
     rc, re = self:_linklast(res, return_flags)
     if not rc then
         return false, e:cat(re)
     end
-    -- return true
+
     return_flags.message = e2lib.align(columns,
         0, string.format("skipping %-20s", res:get_name()),
         columns, string.format("[%s]", sbid))
@@ -354,27 +355,24 @@ function e2build.build_process_class:_setup_chroot(res, return_flags)
         return false, e:cat(re)
     end
 
-    local grp
+    local grp, path
     for cgrpnm in res:my_chroot_list():iter_sorted() do
         grp = chroot.groups_byname[cgrpnm]
 
         for f in grp:file_iter() do
-            local flags = { cache = true }
-            rc, re = cache.cache_file(info.cache, f.server,
-                f.location, flags)
-            if not rc then
-                return false, e:cat(re)
-            end
-            local path, re = cache.file_path(info.cache, f.server, f.location)
+
+            path, re = cache.fetch_file_path(info.cache, f.server, f.location)
             if not path then
                 return false, e:cat(re)
             end
+
             if f.sha1 then
                 rc, re = e2tool.verify_hash(info, f.server, f.location, f.sha1)
                 if not rc then
                     return false, e:cat(re)
                 end
             end
+
             local tartype
             tartype, re = e2lib.tartype_by_suffix(path)
             if not tartype then
@@ -553,7 +551,7 @@ function e2build.build_process_class:helper_unpack_result(res, dep, destdir)
         dep:get_name(), server, location)
 
     resulttarpath = e2lib.join(location, dep:get_name(), dep_set, "result.tar")
-    path, re = cache.file_path(info.cache, server, resulttarpath)
+    path, re = cache.fetch_file_path(info.cache, server, resulttarpath)
     if not path then
         return false, e:cat(re)
     end
@@ -829,7 +827,7 @@ function e2build.build_process_class:helper_deploy(res, tmpdir)
             return false, re
         end
     end
-    if cache.writeback_state(info.cache, server, cache_flags) == false then
+    if cache.writeback_enabled(info.cache, server, cache_flags) == false then
         e2lib.warnf("WOTHER",
             "Writeback is disabled for server %q. Release not deployed!", server)
     end
@@ -959,41 +957,67 @@ end
 
 --- TODO
 function e2build.build_process_class:_linklast(res, return_flags)
-    local rc, re, info
-    local e = err.new("creating link to last results")
+    local rc, re, e
+    local info, server, location, buildid, dst, lnk
+    
+    e = err.new("creating link to last results")
     info = e2tool.info()
     -- calculate the path to the result
-    local server, location = res:build_mode().storage(info.project_location,
-        project.release_id())
+    server, location = res:build_mode().storage(info.project_location, project.release_id())
 
-    local buildid, re = res:buildid()
+    -- compute the "last" link/directory
+    buildid, re = res:buildid()
     if not buildid then
         return false, e:cat(re)
     end
-    local location1 = e2lib.join(location, res:get_name(), buildid)
-    local dst, re = cache.file_path(info.cache, server, location1)
-    if not dst then
-        return false, e:cat(re)
-    end
-    -- create the last link
-    local lnk_location = e2lib.join("out", res:get_name(), "last")
-    local lnk, re = cache.file_path(info.cache, info.root_server_name, lnk_location)
-    if not lnk then
-        return false, e:cat(re)
-    end
-    rc, re = e2lib.mkdir_recursive(e2lib.dirname(lnk))
-    if not rc then
-        return false, e:cat(re)
-    end
+    lnk = e2lib.join(info.root,  "out", res:get_name(), "last")
+    location = e2lib.join(location, res:get_name(), buildid, "result.tar")
 
-    if e2lib.stat(lnk, false) then
-        e2lib.unlink(lnk) -- ignore errors, symlink will catch it
-    end
+    -- if we don't have cache or server on local fs, fetch a copy into "out"
+    if not cache.cache_enabled(info.cache, server) and not
+        cache.islocal_enabled(info.cache, server) then
+        e2lib.logf(3, "%s: copy to out/%s/last, server %q has no cache/not local",
+            res:get_name(), res:get_name(), server)
 
-    rc, re = e2lib.symlink(dst, lnk)		-- create the new link
-    if not rc then
-        return false, e:cat(re)
+        if e2lib.stat(lnk, false) then
+            e2lib.unlink_recursive(lnk) -- ignore errors
+        end
+
+        rc, re = e2lib.mkdir_recursive(lnk)
+        if not rc then
+            return e:cat(re)
+        end
+
+        rc, re = cache.fetch_file(info.cache, server, location, lnk, nil)
+        if not rc then
+            return false, e:cat(re)
+        end
+
+        return true
+    else -- otherwise create a symlink
+        dst, re = cache.fetch_file_path(info.cache, server, location)
+        if not dst then
+            return false, e:cat(re)
+        end
+
+        dst = e2lib.dirname(dst) -- we only care about the directory
+
+        -- create the last link
+        rc, re = e2lib.mkdir_recursive(e2lib.dirname(lnk))
+        if not rc then
+            return false, e:cat(re)
+        end
+
+        if e2lib.stat(lnk, false) then
+            e2lib.unlink_recursive(lnk) -- ignore errors, symlink will catch it
+        end
+
+        rc, re = e2lib.symlink(dst, lnk)
+        if not rc then
+            return false, e:cat(re)
+        end
     end
+
     return true
 end
 
diff --git a/local/e2tool.lua b/local/e2tool.lua
index b1eb649..e4e5596 100644
--- a/local/e2tool.lua
+++ b/local/e2tool.lua
@@ -895,7 +895,6 @@ end
 -- @return an error object on failure
 function e2tool.fileid(info, file)
     local rc, re, e, fileid, path
-    local cache_flags = { cache = true }
 
     e = err.new("error calculating file id for file: %s:%s",
         file.server, file.location)
@@ -903,13 +902,7 @@ function e2tool.fileid(info, file)
     if file.sha1 then
         fileid = file.sha1
     else
-        rc, re = cache.cache_file(info.cache, file.server,
-            file.location, cache_flags)
-        if not rc then
-            return false, e:cat(re)
-        end
-
-        path, re = cache.file_path(info.cache, file.server, file.location)
+        path, re = cache.fetch_file_path(info.cache, file.server, file.location)
         if not path then
             return false, e:cat(re)
         end
diff --git a/local/policy.lua b/local/policy.lua
index cdab23e..6456ef7 100644
--- a/local/policy.lua
+++ b/local/policy.lua
@@ -191,16 +191,13 @@ function policy.init(info)
         local ce, re = cache.ce_by_server(info.cache, server)
         if not ce then
             se:cat(re)
-        elseif not ce.flags.writeback then
+        elseif not cache.writeback_enabled(info.cache, server) then
             e2lib.warnf("WPOLICY",
             "Results will not be pushed to server: '%s'"..
             " (Writeback disabled)", server)
         end
-        if ce and not (ce.flags.cache or ce.flags.islocal) then
-            se:append(
-            "Building needs local access to build results. "..
-            "Enable cache.")
-        elseif ce and not (ce.flags.writeback or ce.flags.cache) then
+        if ce and not (cache.writeback_enabled(info.cache, server)
+            or cache.cache_enabled(info.cache, server)) then
             se:append(
             "Cannot store results. "..
             "Enable cache or writeback.")
diff --git a/plugins/files.lua b/plugins/files.lua
index 70af7c0..ec624be 100644
--- a/plugins/files.lua
+++ b/plugins/files.lua
@@ -348,16 +348,15 @@ function files.cache_source(info, sourcename)
 
     -- cache all files for this source
     for f in src:file_iter() do
-        e2lib.logf(4, "files.cache_source: caching file %s:%s", f.server,
-            f.location)
-        local flags = { cache = true }
-        if f.server ~= info.root_server_name then
-            rc, re = cache.cache_file(info.cache, f.server, f.location, flags)
+        if cache.cache_enabled(info.cache, f.server) then
+            e2lib.logf(3, "files.cache_source: caching file %s:%s",
+                f.server, f.location)
+            rc, re = cache.fetch_file_path(info.cache, f.server, f.location)
             if not rc then
                 return false, re
             end
         else
-            e2lib.logf(4, "not caching %s:%s (stored locally)", f.server,
+            e2lib.logf(3, "not caching %s:%s (stored locally)", f.server,
                 f.location)
         end
     end
@@ -548,13 +547,8 @@ function files.prepare_source(info, sourcename, sourceset, buildpath)
             end
         end
         if file.unpack then
-            local cache_flags = { cache = true }
-            local rc, re = cache.cache_file(info.cache, file.server,
-                file.location, cache_flags)
-            if not rc then
-                return false, e:cat(re)
-            end
-            local path, re = cache.file_path(info.cache, file.server, file.location)
+            local path, re = cache.fetch_file_path(info.cache,
+                file.server, file.location)
             if not path then
                 return false, e:cat(re)
             end
@@ -597,13 +591,8 @@ function files.prepare_source(info, sourcename, sourceset, buildpath)
                 end
             end
             if file.patch then
-                local cache_flags = { cache = true }
-                local rc, re = cache.cache_file(info.cache, file.server,
-                    file.location, cache_flags)
-                if not rc then
-                    return false, e:cat(re)
-                end
-                local path, re = cache.file_path(info.cache, file.server, file.location)
+                local path, re = cache.fetch_file_path(info.cache,
+                    file.server, file.location)
                 if not path then
                     return false, e:append(re)
                 end