Symfony ESI breaks Varnish cache if cookies are enabled

Question

I don't know what I am doing anymore. I had so many issues I don't know where to start. Here is my configuration:

varnishd (varnish-3.0.3 revision 9e6a70f)
Server version: Apache/2.2.22 (Unix)
Symfony 2.3.1

First I've disabled Symfony AppCache in the app.php file which was used as a reverse proxy instead of Varnish.

Here is my Varnish configuration:

Varnish (80) <--> Apache (8080)

# /etc/varnish/default.vcl
backend default {
    .host = "127.0.0.1";
    .port = "8080";
}

sub vcl_recv {
    if (req.http.Cache-Control ~ "no-cache") {
        return (pass);
    }

    if (!(req.url ~ "^/dashboard/")) {
        unset req.http.Cookie;
    }

    # Remove has_js and Google Analytics __* cookies.
    set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
    # Remove a ";" prefix, if present.
    set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");

    #set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(__[a-z]+|has_js)=[^;]*", "");
    set req.http.Surrogate-Capability = "abc=ESI/1.0";

    if (req.restarts == 0) {
        if (req.http.x-forwarded-for) {
            set req.http.X-Forwarded-For =
            req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    if (req.request != "GET" &&
        req.request != "HEAD" &&
        req.request != "PUT" &&
        req.request != "POST" &&
        req.request != "TRACE" &&
        req.request != "OPTIONS" &&
        req.request != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }

f (req.request != "GET" && req.request != "HEAD") {
        /* We only deal with GET and HEAD by default */
        return (pass);
    }

    if (req.http.Authorization) {
        /* Not cacheable by default */
        return (pass);
    }

    return (lookup);
}

sub vcl_pipe {
     return (pipe);
}

sub vcl_pass {
    return (pass);
}

sub vcl_hash {
    hash_data(req.url);

    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    return (hash);
}

sub vcl_fetch {
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }

    # Varnish determined the object was not cacheable
    if (beresp.ttl <= 0s) {
        set beresp.http.X-Varnish-Cacheable = "NO:Not Cacheable";

    # You don't wish to cache content for logged in users
    } elsif (req.http.Cookie ~ "(UserID|_session)") {
        set beresp.http.X-Varnish-Cacheable = "NO:Got Session";
        return(hit_for_pass);

    # You are respecting the Cache-Control=private header from the backend
    } elsif (beresp.http.Cache-Control ~ "private") {
        set beresp.http.X-Varnish-Cacheable = "NO:Cache-Control=private";
        return(hit_for_pass);

    # Varnish determined the object was cacheable
    } else {
        set beresp.http.X-Varnish-Cacheable = "YES";
    }

    if (beresp.status >= 300) {
        return (hit_for_pass);
    }

    if (beresp.http.Pragma ~ "no-cache" ||
        beresp.http.Cache-Control ~ "no-cache" ||
        beresp.http.Cache-Control ~ "private") {
        return (hit_for_pass);
    }

    return (deliver);
}

sub vcl_hit {
    return (deliver);
}

sub vcl_deliver {
    if (obj.hits > 0) {
        set resp.http.X-Varnish-Cached = "HIT";
    } else {
        set resp.http.X-Varnish-Cached = "MISS";
    }
    return (deliver);
}

sub vcl_error {
    set obj.http.Content-Type = "text/html; charset=utf-8";
    set obj.http.Retry-After = "5";

    synthetic {"
 <?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html>
   <head>
     <title>"} + obj.status + " " + obj.response + {"</title>
   </head>
   <body>
     <h1>Error "} + obj.status + " " + obj.response + {"</h1>
     <p>"} + obj.response + {"</p>
     <h3>Guru Meditation:</h3>
     <p>XID: "} + req.xid + {"</p>
     <hr>
     <p>Varnish cache server</p>
   </body>
 </html>
 "};

    return (deliver);
}

sub vcl_init {
    return (ok);
}

sub vcl_fini {
    return (ok);
}

Because of:

if (!(req.url ~ "^/dashboard/")) {
     unset req.http.Cookie;
}

Varnish removes all the Cookies of the public pages and HITs the Cache, which is good but... Since Varnish deletes all the cookies I cannot manage to stay logged in (no session cookie means no session).

I have the login button in an ESI block with no-cache header but this still does not fix it. Here is the frontend controller:

public function indexAction()
{
    $response = $this->render('AcmeCoreBundle:Default:index.html.twig');
    $response->setSharedMaxAge(21600); // 6 hours
    return $response;
}

The template extends the layout.html.twig :

<html lang="en-US">
<head>
    ...
</head>
<body>
    ...

    <div class="top-right">
        {{ render_esi(controller('AcmeUserBundle:Default:loginBox')) }}
    </div>

    ...
</body>

and the controller AcmeUserBundle:Default:loginBox:

public function loginBoxAction()
{
    $response = $this->render('AcmeUserBundle:Block:home_login.html.twig');
    $response->setVary('Cookies', false);
    $response->setMaxAge(0);
    $response->setPrivate();

    return $response;
}

I don't know how to manage the session cookie for Symfony. Because I have a facebook connect button on every page I need to have the user session. And because Symfony is creating a cookie even for anonymous user I have session cookies on all requests.

Help would be greatly appreciated :)

Thanks, Maxime

UPDATE: New VCL files after recommendations:

...

sub vcl_recv {

   if (!(req.url ~ "^/dashboard") && !(req.url ~ "^/logout") && !(req.url ~ "^/_fragment") && req.esi_level == 0 ) {
       set req.http.Esi-Cookie = req.http.Cookie;
       unset req.http.Cookie;
   }

   if (!(req.url ~ "^/dashboard") && req.esi_level > 0 ) {
       set req.http.Cookie = req.http.Esi-Cookie;
   }

    # Remove has_js and Google Analytics __* cookies.
    set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");

    # Remove a ";" prefix, if present.
    set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");

    # Force ESI capability header
    set req.http.Surrogate-Capability = "abc=ESI/1.0";

    if (req.restarts == 0) {
        if (req.http.x-forwarded-for) {
            set req.http.X-Forwarded-For =
            req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    if (req.request != "GET" &&
        req.request != "HEAD" &&
        req.request != "PUT" &&
        req.request != "POST" &&
        req.request != "TRACE" &&
        req.request != "OPTIONS" &&
        req.request != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }

    # if Authorization or no-cache header we skip the cache
    if (req.http.Authorization || req.http.Cache-Control ~ "no-cache") {
        return (pass);
    }

    # If not GET or HEAD request we skip the cache
    if (req.request != "GET" && req.request != "HEAD") {
        return (pass);
    }

    return (lookup);
}

...

sub vcl_fetch {

    if (!(req.url ~ "^/dashboard") && !(req.url ~ "^/login_check")) {
        unset beresp.http.set-cookie;
    }

    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }

    # Varnish determined the object was not cacheable
    if (beresp.ttl <= 0s) {
        set beresp.http.X-Varnish-Cacheable = "NO:Not Cacheable";

    # You don't wish to cache content for logged in users
    } elsif (req.http.Cookie ~ "(UserID|_session)") {
        set beresp.http.X-Varnish-Cacheable = "NO:Got Session";
        return(hit_for_pass);

    # You are respecting the Cache-Control=private header from the backend
    } elsif (beresp.http.Cache-Control ~ "private") {
        set beresp.http.X-Varnish-Cacheable = "NO:Cache-Control=private";
        return(hit_for_pass);

    # Varnish determined the object was cacheable
    } else {
        set beresp.http.X-Varnish-Cacheable = "YES";
    }

    if (beresp.status >= 300) {
        return (hit_for_pass);
    }

    if (beresp.http.Pragma ~ "no-cache" ||
        beresp.http.Cache-Control ~ "no-cache" ||
        beresp.http.Cache-Control ~ "private") {
        return (hit_for_pass);
    }

    return (deliver);
}

Every public page is now cached properly. I have cache HITs everywhere excepti on the login page but it's not a big deal for now.

The problem I have is with the ESI block in the header. I can see in the apache access log that varnish is requesting the <esi:include> calling the /_fragment url.

The ESI block rendered on the home page is not correct (it's displaying login url as if the user is not logged in) however if I call the /_fragment directly, the returned block is correct (with the user information).

I don't really know where it's coming from but I'm so close :)

Johnny C · Answer 1 · 2013-07-18T14:40:01.197

5

Removing the Cookie header from an inbound request removes it from all resulting ESI include requests. Since you want access to the cookie header in included resources, but not the parent, which is cached, try this:

if (!(req.url ~ "^/dashboard/") && req.esi_level == 0 ) {
    set req.http.Esi-Cookie = req.http.Cookie;
    unset req.http.Cookie;
}
if (!(req.url ~ "^/dashboard/") && req.esi_level > 0 ) {
    set req.http.Cookie = req.http.Esi-Cookie;
}

This strips the browser cookie from the request for the parent page, but re-adds it to esi requests resulting from esi:include tags in the returned page. I didn't lint the code above so it may not be 100% perfect.

Update

In vcl_recv, if you don't ever want to cache or recursively esi process an esi fragment, you can change the second if block to:

if (!(req.url ~ "^/dashboard/") && req.esi_level > 0 ) {
    set req.http.Cookie = req.http.Esi-Cookie;
    return (pass);
}

edited Jul 18 '13 at 14:40

answered Jul 17 '13 at 23:42

Johnny C

1,799
1
16
27

1

Thanks mate. Since Symfony is creating a session cookie even for anonymous users, do you think creating a second cookie for logged in users could be the solution? If `logged_in` cookie present do not unset cookie, else unset cookies. This would result in Cache MISS for logged in users and Cache HIT for anonymous users. What do you think? – maxwell2022 Jul 18 '13 at 00:30
1

Inspect the cookie header of the requests made to varnish to make sure your VCL is properly setting it. Also when hitting the /_fragment resource directly, check the response headers to make sure they're explicitly uncacheable (or return (pass) for the esi requests as I suggest in the update) – Johnny C Jul 18 '13 at 14:40

Symfony ESI breaks Varnish cache if cookies are enabled

1 Answers1