Install Magento 2 and Varnish Cache on different server

Question

I have 2 servers, one which has magento 2 installed (ip - 129.89.188.244 port 80) and Varnish on another (ip - 129.89.188.245 port 80)

My Varnish Configuration:

File /etc/default/varnish:-

DAEMON_OPTS="-a :80 \
             -T 127.0.0.1:6082 \
             -b 129.89.188.244:80 \
             -f /etc/varnish/default.vcl \
             -S /etc/varnish/secret \
             -s malloc,256m"

netstat -tulpn :-

tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      1288/sshd       
tcp        0      0 127.0.0.1:6082          0.0.0.0:*               LISTEN      11115/varnishd  
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      11115/varnishd  
tcp6       0      0 :::22                   :::*                    LISTEN      1288/sshd       
tcp6       0      0 :::80                   :::*                    LISTEN      11115/varnishd

/etc/varnish/default.vcl : -

# VCL version 5.0 is not supported so it should be 4.0 even though actually used Varnish version is 5
vcl 4.0;

import std;
# The minimal Varnish version is 5.0
# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https'

backend default {
    .host = "129.89.188.244";
    .port = "80";
    .first_byte_timeout = 600s;
    .probe = {
        .url = "/pub/health_check.php";
        .timeout = 2s;
        .interval = 5s;
        .window = 10;
        .threshold = 5;
   }
}

acl purge {
    "129.89.188.245";
    "127.0.0.1";
    "localhost";
}

sub vcl_recv {
    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405, "Method not allowed"));
        }
        # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
        # has been added to the response in your backend server config. This is used, for example, by the
        # capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
        if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
            return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
        }
        if (req.http.X-Magento-Tags-Pattern) {
          ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
        }
        if (req.http.X-Pool) {
          ban("obj.http.X-Pool ~ " + req.http.X-Pool);
        }
        return (synth(200, "Purged"));
    }

    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "DELETE") {
          /* Non-RFC2616 or CONNECT which is weird. */
          return (pipe);
    }

    # We only deal with GET and HEAD by default
    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }

    # Bypass shopping cart, checkout and search requests
    if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") {
        return (pass);
    }

    # Bypass health check requests
    if (req.url ~ "/pub/health_check.php") {
        return (pass);
    }

    # Set initial grace period usage status
    set req.http.grace = "none";

    # normalize url in case of leading HTTP scheme and domain
    set req.url = regsub(req.url, "^http[s]?://", "");

    # collect all cookies
    std.collect(req.http.Cookie);

    # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
            # No point in compressing these
            unset req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            unset req.http.Accept-Encoding;
        }
    }

    # Remove Google gclid parameters to minimize the cache objects
    set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
    set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
    set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"

    # Static files caching
    if (req.url ~ "^/(pub/)?(media|static)/") {
        # Static files should not be cached by default
        return (pass);

        # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
        #unset req.http.Https;
        #unset req.http.X-Forwarded-Proto;
        #unset req.http.Cookie;
    }

    return (hash);
}

sub vcl_hash {
    if (req.http.cookie ~ "X-Magento-Vary=") {
        hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
    }

    # For multi site configurations to not cache each other's content
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    # To make sure http users don't see ssl warning
    if (req.http.X-Forwarded-Proto) {
        hash_data(req.http.X-Forwarded-Proto);
    }

}

sub vcl_backend_response {

    set beresp.grace = 3d;

    if (beresp.http.content-type ~ "text") {
        set beresp.do_esi = true;
    }

    if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
        set beresp.do_gzip = true;
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # cache only successfully responses and 404s
    if (beresp.status != 200 && beresp.status != 404) {
        set beresp.ttl = 0s;
        set beresp.uncacheable = true;
        return (deliver);
    } elsif (beresp.http.Cache-Control ~ "private") {
        set beresp.uncacheable = true;
        set beresp.ttl = 86400s;
        return (deliver);
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
        unset beresp.http.set-cookie;
    }

   # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
   if (beresp.ttl <= 0s ||
       beresp.http.Surrogate-control ~ "no-store" ||
       (!beresp.http.Surrogate-Control &&
       beresp.http.Cache-Control ~ "no-cache|no-store") ||
       beresp.http.Vary == "*") {
        # Mark as Hit-For-Pass for the next 2 minutes
        set beresp.ttl = 120s;
        set beresp.uncacheable = true;
    }

    return (deliver);
}

sub vcl_deliver {
    if (resp.http.X-Magento-Debug) {
        if (resp.http.x-varnish ~ " ") {
            set resp.http.X-Magento-Cache-Debug = "HIT";
            set resp.http.Grace = req.http.grace;
        } else {
            set resp.http.X-Magento-Cache-Debug = "MISS";
        }
    } else {
        unset resp.http.Age;
    }

    # Not letting browser to cache non-static files.
    if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
        set resp.http.Pragma = "no-cache";
        set resp.http.Expires = "-1";
        set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
    }

    unset resp.http.X-Magento-Debug;
    unset resp.http.X-Magento-Tags;
    unset resp.http.X-Powered-By;
    unset resp.http.Server;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
        # Hit within TTL period
        return (deliver);
    }
    if (std.healthy(req.backend_hint)) {
        if (obj.ttl + 300s > 0s) {
            # Hit after TTL expiration, but within grace period
            set req.http.grace = "normal (healthy server)";
            return (deliver);
        } else {
            # Hit after TTL and grace expiration
            return (miss);
        }
    } else {
        # server is not healthy, retrieve from cache
        set req.http.grace = "unlimited (unhealthy server)";
        return (deliver);
    }
}

Now the issue is, when I open the URL 129.89.188.244 , magento opens but it's not getting cached in varnish. But when I call varnish URL 129.89.188.245, it will redirect to my magento url 129.89.188.244. In my varnish log, it shows that the page is cached already but magento is not getting served from that varnish cache.

I am not sure but this could be related [issue](https://stackoverflow.com/questions/25843279/how-to-set-up-varnish-on-different-server) — Anouar Kacem, Mar 24 '18 at 18:21
Nope, his backend points to the correct host - see: backend default { .host = "129.89.188.244"; in his listed configuration file. — hovanessyan, Mar 24 '18 at 18:23
Can you post 2 screenshots of the requested cachable resources in the web/dev console, of 2 consecutive requests made from the same browser against the Varnish URL? It would be best if you can clear the Varnish cache before that. — hovanessyan, Mar 24 '18 at 18:27
When you are testing this - have you set magento into dev mode? magento deploy:mode:set — hovanessyan, Mar 24 '18 at 18:34
@DhavalBhavsar, can you please add the output of `curl -v http://129.89.188.245` to your question? — Tarun Lalwani, Mar 25 '18 at 06:25

hovanessyan · Answer 1 · 2018-03-24T18:35:34.083

3

In order for the cache to work, your requests always have to go through the proxy/cache (Varnish).

Varnish evaluates your requests and decides if the request is already cached (then returns it from the cache) or not - then it will redirect it to Magento and cache the response, before returning it to the client.

That's how most caches work. You can read in detail how Varnish cache mechanism works here.

If you want to hit your cache, you always have to go through Varnish (129.89.188.245:80).

Consider this diagram form the official documentation:

edited Mar 24 '18 at 18:35

answered Mar 24 '18 at 18:18

hovanessyan

30,580
6
55
83

so you mean that i need to call varnish server that's right but varnish redirect to magento server in these case what i need to do? – Dhaval Bhavsar Mar 25 '18 at 04:50
Yes you need to call the Varnish server, not calling the Magento server directly. I think your setup is correct. You can verify if you're getting back cached results following the verification steps as described here: http://devdocs.magento.com/guides/v2.0/config-guide/varnish/config-varnish-final.html – hovanessyan Mar 25 '18 at 10:49
But i have one question if i will call varnish server but it will redirect magento then after that i click on any link in my e-commerce cart it's not call varnish so for that what i need to do? – Dhaval Bhavsar Mar 28 '18 at 05:53
You should fix magento base url and clear cache by the commands below: ``` bin/magento setup:store-config:set --base-url="http://varnish.host/" bin/magento setup:store-config:set --base-url-secure="https://varnish.host/" bin/magento cache:flush ``` – likemusic Nov 27 '19 at 22:21
how can I check the diagram working fine on my server? – huykon225 Feb 26 '21 at 08:56

score 0 · Answer 2 · answered Mar 19 '18 at 10:09

this is the expected behaviour. Varnish sits in front of you Magento server and if you ping your Magento server bypassing Varnish, opening 129.89.188.244, then Magento will send you a response without involving Varnish, while if you ping varnish, Varnish will then call Magento and cache. The other way around is not possible and makes no sense.

score 0 · Answer 3 · answered Apr 01 '22 at 10:44

This solution worked for me. To configure Varnish and Magento on a different server.

Varnish server: xxx.xxx.xxx.xxx port 80

Magento server: yyy.yyy.yyy.yyy port 80

Changes need to be made on the varnish server: 

 1. login to varnish server
 2. go to file /etc/varnish/default.vcl
 3. under the "backend default" update

    .host = "yyy.yyy.yyy.yyy";//(use Magento server IP for better network)
    .port = "80";//(Magento web server port) 
 4. Restart the Varnish (systemctl restart varnish)

Note: Kindly use the default VCL that is generated during the varnish installation and don't update it with Magento generated VCL for varnish ( available from Magento Admin)

Changes need to be made on the Magento server:
1. Log in to the Magento server
2. Go to the env.php file located in the app/etc directory
3. Update the values in     'http_cache_hosts' => [
        [
            'host' => 'xxx.xxx.xxx.xxx', //(varnish server public ip)
            'port' => '80' // ( varnish server port)
        ]
    ]
Now update your base URL's on the core_config_data table to your varnish public  Ip (http://xxx.xxx.xxx.xxx/)
flush the Magento caches ( bin/magento ca:fl)

Install Magento 2 and Varnish Cache on different server

3 Answers3