/* * $Id: refresh.c,v 1.67.2.1 2009/06/25 22:55:50 hno Exp $ * * DEBUG: section 22 Refresh Calculation * AUTHOR: Harvest Derived * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- * * Squid is the result of efforts by numerous individuals from * the Internet community; see the CONTRIBUTORS file for full * details. Many organizations have provided support for Squid's * development; see the SPONSORS file for full details. Squid is * Copyrighted (C) 2001 by the Regents of the University of * California; see the COPYRIGHT file for full details. Squid * incorporates software developed and/or copyrighted by other * sources; see the CREDITS file for full details. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #ifndef USE_POSIX_REGEX #define USE_POSIX_REGEX /* put before includes; always use POSIX */ #endif #include "squid.h" typedef enum { rcHTTP, rcICP, #if USE_HTCP rcHTCP, #endif #if USE_CACHE_DIGESTS rcCDigest, #endif rcStore, rcCount } refreshCountsEnum; typedef struct { unsigned int expires:1; unsigned int min:1; unsigned int lmfactor:1; unsigned int max; } stale_flags; /* * This enumerated list assigns specific values, ala HTTP/FTP status * codes. All Fresh codes are in the range 100-199 and all stale * codes are 200-299. We might want to use these codes in logging, * so best to keep them consistent over time. */ enum { FRESH_REQUEST_MAX_STALE_ALL = 100, FRESH_REQUEST_MAX_STALE_VALUE, FRESH_EXPIRES, FRESH_LMFACTOR_RULE, FRESH_MIN_RULE, FRESH_OVERRIDE_EXPIRES, FRESH_OVERRIDE_LASTMOD, STALE_MUST_REVALIDATE = 200, STALE_RELOAD_INTO_IMS, STALE_FORCED_RELOAD, STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE, STALE_EXPIRES, STALE_MAX_RULE, STALE_LMFACTOR_RULE, STALE_WITHIN_DELTA, STALE_ASYNC_REFRESH, STALE_MAX_STALE, STALE_DEFAULT = 299 }; static struct RefreshCounts { const char *proto; int total; int status[STALE_DEFAULT + 1]; } refreshCounts[rcCount]; /* * Defaults: * MIN NONE * PCT 20% * MAX 3 days */ #define REFRESH_DEFAULT_MIN (time_t)0 #define REFRESH_DEFAULT_PCT 0.20 #define REFRESH_DEFAULT_MAX (time_t)259200 static const refresh_t *refreshUncompiledPattern(const char *); static OBJH refreshStats; static int refreshStaleness(const StoreEntry *, time_t, time_t, const refresh_t *, stale_flags *); static refresh_t DefaultRefresh; const refresh_t * refreshLimits(const char *url) { const refresh_t *R; for (R = Config.Refresh; R; R = R->next) { if (!regexec(&(R->compiled_pattern), url, 0, 0, 0)) return R; } return NULL; } static const refresh_t * refreshUncompiledPattern(const char *pat) { const refresh_t *R; for (R = Config.Refresh; R; R = R->next) { if (0 == strcmp(R->pattern, pat)) return R; } return NULL; } refresh_cc refreshCC(const StoreEntry * entry, request_t * request) { const refresh_t *R; const char *uri = NULL; refresh_cc cc; cc.negative_ttl = Config.negativeTtl; cc.max_stale = Config.maxStale; if (entry->mem_obj) uri = entry->mem_obj->url; else if (request) uri = urlCanonical(request); R = uri ? refreshLimits(uri) : refreshUncompiledPattern("."); if (R && R->negative_ttl >= 0) cc.negative_ttl = R->negative_ttl; if (R && R->max_stale >= 0) cc.max_stale = R->max_stale; return cc; } /* * Calculate how stale the response is (or will be at the check_time). * Staleness calculation is based on the following: (1) response * expiration time, (2) age greater than configured maximum, (3) * last-modified factor, and (4) age less than configured minimum. * * If the response is fresh, return -1. Otherwise return its * staleness. NOTE return value of 0 means the response is stale. * * The 'stale_flags' structure is used to tell the calling function * _why_ this response is fresh or stale. Its used, for example, * when the admin wants to override expiration and last-modified * times. */ static int refreshStaleness(const StoreEntry * entry, time_t check_time, time_t age, const refresh_t * R, stale_flags * sf) { /* * Check for an explicit expiration time. */ if (entry->expires > -1) { sf->expires = 1; if (entry->expires > check_time) { debug(22, 3) ("FRESH: expires %d >= check_time %d \n", (int) entry->expires, (int) check_time); return -1; } else { debug(22, 3) ("STALE: expires %d < check_time %d \n", (int) entry->expires, (int) check_time); return (check_time - entry->expires); } } assert(age >= 0); /* * Use local heuristics to determine staleness. Start with the * max age from the refresh_pattern rule. */ if (age > R->max) { debug(22, 3) ("STALE: age %d > max %d \n", (int) age, (int) R->max); sf->max = 1; return (age - R->max); } if (check_time < entry->timestamp) { debug(22, 1) ("STALE: Entry's timestamp greater than check time. Clock going backwards?\n"); debug(22, 1) ("\tcheck_time:\t%s\n", mkrfc1123(check_time)); debug(22, 1) ("\tentry->timestamp:\t%s\n", mkrfc1123(entry->timestamp)); debug(22, 1) ("\tstaleness:\t%ld\n", (long int) entry->timestamp - check_time); return (entry->timestamp - check_time); } /* * Try the last-modified factor algorithm. */ if (entry->lastmod > -1 && entry->timestamp > entry->lastmod) { /* * stale_age is the Age of the response when it became/becomes * stale according to the last-modified factor algorithm. */ time_t stale_age = (entry->timestamp - entry->lastmod) * R->pct; sf->lmfactor = 1; if (age >= stale_age) { debug(22, 3) ("STALE: age %d > stale_age %d\n", (int) age, (int) stale_age); return (age - stale_age); } else { debug(22, 3) ("FRESH: age %d <= stale_age %d\n", (int) age, (int) stale_age); return -1; } } /* * If we are here, staleness is determined by the refresh_pattern * configured minimum age. */ if (age < R->min) { debug(22, 3) ("FRESH: age %d < min %d\n", (int) age, (int) R->min); sf->min = 1; return -1; } debug(22, 3) ("STALE: age %d >= min %d\n", (int) age, (int) R->min); return (age - R->min); } /* return 1 if the entry must be revalidated within delta seconds * 0 otherwise * * note: request maybe null (e.g. for cache digests build) */ static int refreshCheck(const StoreEntry * entry, request_t * request, time_t delta) { const refresh_t *R; const char *uri = NULL; time_t age = 0; time_t check_time = squid_curtime; int staleness; stale_flags sf; if (entry->mem_obj) uri = entry->mem_obj->url; else if (request) uri = urlCanonical(request); debug(22, 3) ("refreshCheck: '%s'\n", uri ? uri : ""); if (delta > 0) check_time += delta; if (check_time > entry->timestamp) age = check_time - entry->timestamp; R = uri ? refreshLimits(uri) : refreshUncompiledPattern("."); if (NULL == R) R = &DefaultRefresh; memset(&sf, '\0', sizeof(sf)); staleness = refreshStaleness(entry, check_time, age, R, &sf); debug(22, 3) ("Staleness = %d\n", staleness); debug(22, 3) ("refreshCheck: Matched '%s %d %d%% %d'\n", R->pattern, (int) R->min, (int) (100.0 * R->pct), (int) R->max); debug(22, 3) ("refreshCheck: age = %d\n", (int) age); debug(22, 3) ("\tcheck_time:\t%s\n", mkrfc1123(check_time)); debug(22, 3) ("\tentry->timestamp:\t%s\n", mkrfc1123(entry->timestamp)); if (EBIT_TEST(entry->flags, ENTRY_REVALIDATE) && staleness > -1) { debug(22, 3) ("refreshCheck: YES: Must revalidate stale response\n"); return STALE_MUST_REVALIDATE; } /* request-specific checks */ if (request) { HttpHdrCc *cc = request->cache_control; #if HTTP_VIOLATIONS if (!request->flags.nocache_hack) { (void) 0; } else if (R->flags.ignore_reload) { /* The clients no-cache header is ignored */ debug(22, 3) ("refreshCheck: MAYBE: ignore-reload\n"); } else if (R->flags.reload_into_ims || Config.onoff.reload_into_ims) { /* The clients no-cache header is changed into a IMS query */ debug(22, 3) ("refreshCheck: YES: reload-into-ims\n"); return STALE_RELOAD_INTO_IMS; } else { /* The clients no-cache header is not overridden on this request */ debug(22, 3) ("refreshCheck: YES: client reload\n"); request->flags.nocache = 1; return STALE_FORCED_RELOAD; } #endif if (NULL != cc) { if (cc->max_age > -1) { #if HTTP_VIOLATIONS if (R->flags.ignore_reload && cc->max_age == 0) { } else #endif if (age > cc->max_age) { debug(22, 3) ("refreshCheck: YES: age > client-max-age\n"); return STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE; } } if (EBIT_TEST(cc->mask, CC_MAX_STALE) && staleness >= 0) { if (cc->max_stale < 0) { /* max-stale directive without a value */ debug(22, 3) ("refreshCheck: NO: max-stale wildcard\n"); return FRESH_REQUEST_MAX_STALE_ALL; } else if (staleness < cc->max_stale) { debug(22, 3) ("refreshCheck: NO: staleness < max-stale\n"); return FRESH_REQUEST_MAX_STALE_VALUE; } } } } if (staleness < 0) { if (sf.expires) return FRESH_EXPIRES; assert(!sf.max); if (sf.lmfactor) return FRESH_LMFACTOR_RULE; assert(sf.min); return FRESH_MIN_RULE; } /* * At this point the response is stale, unless one of * the override options kicks in. */ #if HTTP_VIOLATIONS if (sf.expires && R->flags.override_expire && age < R->min) { debug(22, 3) ("refreshCheck: NO: age < min && override-expire\n"); return FRESH_OVERRIDE_EXPIRES; } if (sf.lmfactor && R->flags.override_lastmod && age < R->min) { debug(22, 3) ("refreshCheck: NO: age < min && override-lastmod\n"); return FRESH_OVERRIDE_LASTMOD; } #endif if (entry->mem_obj) { int stale_while_revalidate = -1; if (entry->mem_obj->reply && entry->mem_obj->reply->cache_control && EBIT_TEST(entry->mem_obj->reply->cache_control->mask, CC_STALE_WHILE_REVALIDATE)) stale_while_revalidate = entry->mem_obj->reply->cache_control->stale_while_revalidate; if (R->flags.ignore_stale_while_revalidate || stale_while_revalidate == -1) stale_while_revalidate = R->stale_while_revalidate; if (staleness < stale_while_revalidate) { debug(22, 3) ("stale-while-revalidate: age=%d, staleness=%d, stale_while_revalidate=%d\n", (int) age, staleness, stale_while_revalidate); entry->mem_obj->stale_while_revalidate = stale_while_revalidate; return STALE_ASYNC_REFRESH; } } { int max_stale = Config.maxStale; if (R->max_stale >= 0) max_stale = R->max_stale; if (entry->mem_obj && entry->mem_obj->reply && entry->mem_obj->reply->cache_control && EBIT_TEST(entry->mem_obj->reply->cache_control->mask, CC_STALE_IF_ERROR)) max_stale = entry->mem_obj->reply->cache_control->stale_if_error; if (max_stale >= 0 && staleness >= max_stale) return STALE_MAX_STALE; } if (delta < 0 && staleness + delta < 0) { return STALE_WITHIN_DELTA; } if (sf.expires) return STALE_EXPIRES; if (sf.max) return STALE_MAX_RULE; if (sf.lmfactor) return STALE_LMFACTOR_RULE; return STALE_DEFAULT; } int refreshIsCachable(const StoreEntry * entry) { /* * Don't look at the request to avoid no-cache and other nuisances. * the object should have a mem_obj so the URL will be found there. * 60 seconds delta, to avoid objects which expire almost * immediately, and which can't be refreshed. */ int reason = refreshCheck(entry, NULL, Config.minimum_expiry_time); int can_revalidate = 0; refreshCounts[rcStore].total++; refreshCounts[rcStore].status[reason]++; if (reason < 200) /* Does not need refresh. This is certainly cachable */ return 1; if (entry->lastmod > 0) can_revalidate = 1; if (entry->mem_obj && entry->mem_obj->reply) { if (httpHeaderHas(&entry->mem_obj->reply->header, HDR_ETAG)) can_revalidate = 1; } /* Last modified is needed to do a refresh */ if (!can_revalidate) return 0; /* This seems to be refreshable. Cache it */ return 1; } /* refreshCheck... functions below are protocol-specific wrappers around * refreshCheck() function above */ int refreshCheckHTTP(const StoreEntry * entry, request_t * request) { int reason = refreshCheck(entry, request, 0); refreshCounts[rcHTTP].total++; refreshCounts[rcHTTP].status[reason]++; return (reason < 200) ? 0 : 1; } int refreshCheckHTTPStale(const StoreEntry * entry, request_t * request) { int reason = refreshCheck(entry, request, -Config.refresh_stale_window); if (reason == STALE_WITHIN_DELTA) return -1; if (reason == STALE_ASYNC_REFRESH) return -2; if (reason == STALE_MAX_STALE) return 3; return (reason < 200) ? 0 : 1; } int refreshCheckStaleOK(const StoreEntry * entry, request_t * request) { int reason = refreshCheck(entry, request, 0); if (reason == STALE_MUST_REVALIDATE) return 0; if (reason == STALE_MAX_STALE) return 0; return 1; } int refreshCheckICP(const StoreEntry * entry, request_t * request) { int reason = refreshCheck(entry, request, 30); refreshCounts[rcICP].total++; refreshCounts[rcICP].status[reason]++; return (reason < 200) ? 0 : 1; } #if USE_HTCP int refreshCheckHTCP(const StoreEntry * entry, request_t * request) { int reason = refreshCheck(entry, request, 10); refreshCounts[rcHTCP].total++; refreshCounts[rcHTCP].status[reason]++; return (reason < 200) ? 0 : 1; } #endif #if USE_CACHE_DIGESTS int refreshCheckDigest(const StoreEntry * entry, time_t delta) { int reason = refreshCheck(entry, entry->mem_obj ? entry->mem_obj->request : NULL, delta); refreshCounts[rcCDigest].total++; refreshCounts[rcCDigest].status[reason]++; return (reason < 200) ? 0 : 1; } #endif time_t getMaxAge(const char *url) { const refresh_t *R; debug(22, 3) ("getMaxAge: '%s'\n", url); if ((R = refreshLimits(url))) return R->max; else return REFRESH_DEFAULT_MAX; } static void refreshCountsStats(StoreEntry * sentry, struct RefreshCounts *rc) { int sum = 0; int tot = rc->total; storeAppendPrintf(sentry, "\n\n%s histogram:\n", rc->proto); storeAppendPrintf(sentry, "Count\t%%Total\tCategory\n"); #define refreshCountsStatsEntry(code,desc) { \ storeAppendPrintf(sentry, "%6d\t%6.2f\t%s\n", \ rc->status[code], xpercent(rc->status[code], tot), desc); \ sum += rc->status[code]; \ } refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_ALL, "Fresh: request max-stale wildcard"); refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_VALUE, "Fresh: request max-stale value"); refreshCountsStatsEntry(FRESH_EXPIRES, "Fresh: expires time not reached"); refreshCountsStatsEntry(FRESH_LMFACTOR_RULE, "Fresh: refresh_pattern last-mod factor percentage"); refreshCountsStatsEntry(FRESH_MIN_RULE, "Fresh: refresh_pattern min value"); refreshCountsStatsEntry(FRESH_OVERRIDE_EXPIRES, "Fresh: refresh_pattern override expires"); refreshCountsStatsEntry(FRESH_OVERRIDE_LASTMOD, "Fresh: refresh_pattern override lastmod"); refreshCountsStatsEntry(STALE_MUST_REVALIDATE, "Stale: response has must-revalidate"); refreshCountsStatsEntry(STALE_RELOAD_INTO_IMS, "Stale: changed reload into IMS"); refreshCountsStatsEntry(STALE_FORCED_RELOAD, "Stale: request has no-cache directive"); refreshCountsStatsEntry(STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE, "Stale: age exceeds request max-age value"); refreshCountsStatsEntry(STALE_EXPIRES, "Stale: expires time reached"); refreshCountsStatsEntry(STALE_MAX_RULE, "Stale: refresh_pattern max age rule"); refreshCountsStatsEntry(STALE_LMFACTOR_RULE, "Stale: refresh_pattern last-mod factor percentage"); refreshCountsStatsEntry(STALE_WITHIN_DELTA, "Staĺe: but within a certain delta"); refreshCountsStatsEntry(STALE_ASYNC_REFRESH, "Stale: suitable for an async refresh"); refreshCountsStatsEntry(STALE_MAX_STALE, "Stale: max-stale"); refreshCountsStatsEntry(STALE_DEFAULT, "Stale: by default"); tot = sum; /* paranoid: "total" line shows 100% if we forgot nothing */ storeAppendPrintf(sentry, "%6d\t%6.2f\tTOTAL\n", rc->total, xpercent(rc->total, tot)); \ storeAppendPrintf(sentry, "\n"); } static void refreshStats(StoreEntry * sentry) { int i; int total = 0; /* get total usage count */ for (i = 0; i < rcCount; ++i) total += refreshCounts[i].total; /* protocol usage histogram */ storeAppendPrintf(sentry, "\nRefreshCheck calls per protocol\n\n"); storeAppendPrintf(sentry, "Protocol\t#Calls\t%%Calls\n"); for (i = 0; i < rcCount; ++i) storeAppendPrintf(sentry, "%10s\t%6d\t%6.2f\n", refreshCounts[i].proto, refreshCounts[i].total, xpercent(refreshCounts[i].total, total)); /* per protocol histograms */ storeAppendPrintf(sentry, "\n\nRefreshCheck histograms for various protocols\n"); for (i = 0; i < rcCount; ++i) refreshCountsStats(sentry, &refreshCounts[i]); } void refreshInit(void) { memset(refreshCounts, 0, sizeof(refreshCounts)); refreshCounts[rcHTTP].proto = "HTTP"; refreshCounts[rcICP].proto = "ICP"; #if USE_HTCP refreshCounts[rcHTCP].proto = "HTCP"; #endif refreshCounts[rcStore].proto = "On Store"; #if USE_CACHE_DIGESTS refreshCounts[rcCDigest].proto = "Cache Digests"; #endif cachemgrRegister("refresh", "Refresh Algorithm Statistics", refreshStats, 0, 1); memset(&DefaultRefresh, '\0', sizeof(DefaultRefresh)); DefaultRefresh.pattern = ""; DefaultRefresh.min = REFRESH_DEFAULT_MIN; DefaultRefresh.pct = REFRESH_DEFAULT_PCT; DefaultRefresh.max = REFRESH_DEFAULT_MAX; }