externallinks/nginx.conf at master · WikipediaLibrary/externallinks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
map $http_x_forwarded_proto $web_proxy_scheme {
  default $scheme;
  https https;
}

map $http_user_agent $limit_bots {
  default "";
  ~*(GoogleBot|bingbot|YandexBot|mj12bot|Apache-HttpClient|Adsbot|Barkrowler|FacebookBot|dotbot|Googlebot|Bytespider|SemrushBot|AhrefsBot|Amazonbot|GPTBot|DotBot) $binary_remote_addr;
}

## Testing the request method
# Only GET and HEAD are caching safe.
map $request_method $no_cache_method {
  default 1;
  HEAD 0;
  GET 0;
}

## Testing for Cache-Control header
# Only checking for no-cache because chrome annoyingly sets max-age=0 when hitting enter in the address bar.
map $http_cache_control $no_cache_control {
  default 0;
  no-cache 1;
}

## Testing for the session cookie being present
map $http_cookie $no_cache_session {
  default 0;
  ~sessionid 1; # Django session cookie
}

## proxy caching settings.
proxy_cache_path /var/lib/nginx/cache levels=1:2 keys_zone=cache:8m max_size=10g inactive=10m;
proxy_cache_key "$scheme$proxy_host$uri$is_args$args$http_accept_language";
proxy_cache_lock on;
proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504;

# remote address is a joke here since we don't have x-forwarded-for
limit_req_zone $limit_bots zone=bots:10m rate=1r/s;
limit_req_zone $binary_remote_addr zone=one:10m rate=500r/s;

upstream django_server {
 server externallinks:8000 fail_timeout=0;
}

server {
  listen 80 deferred;
  client_max_body_size 4G;
  server_name wikilink.wmflabs.org;
  keepalive_timeout 5;

  # Definied explicitly to avoid caching
  location /healthcheck/link_event {
    # Rate limit
    limit_req zone=bots burst=2 nodelay;
    limit_req zone=one burst=1000 nodelay;
    limit_req_status 429;
    # Proxy
    proxy_set_header X-Forwarded-Proto $web_proxy_scheme;
    proxy_set_header Host $http_host;
    proxy_redirect off;
    proxy_pass http://django_server;
  }

  location = /robots.txt {
    add_header Content-Type text/plain;
    alias /app/robots.txt;
  }

  location / {
    root /app/;
    expires 30d;

    if ($http_user_agent ~* (GoogleBot|bingbot|YandexBot|mj12bot|Apache-HttpClient|Adsbot|Barkrowler|FacebookBot|dotbot|Bytespider|SemrushBot|AhrefsBot|Amazonbot|GPTBot) ) {
        return 403;
    }
    location /admin/links/ {
        try_files $uri @django-admin-slow;
    }
    # checks for static file, if not found proxy to app
    try_files $uri @django;
  }

  location @django {
    # Cache
    proxy_cache_valid 200 301 302 401 403 404 1d;
    proxy_cache_bypass $http_pragma $no_cache_method $no_cache_control $no_cache_session;
    proxy_cache_revalidate on;
    proxy_cache cache;
    add_header X-Cache-Status $upstream_cache_status;
    # Rate limit
    limit_req zone=bots burst=2 nodelay;
    limit_req zone=one burst=1000 nodelay;
    limit_req_status 429;
    # Proxy
    proxy_set_header X-Forwarded-Proto $web_proxy_scheme;
    proxy_set_header Host $http_host;
    proxy_redirect off;
    proxy_pass http://django_server;
  }

  location @django-admin-slow {
    # https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout
    proxy_connect_timeout 120s;
    proxy_send_timeout 120s;
    proxy_read_timeout 120s;
    # https://nginx.org/en/docs/http/ngx_http_core_module.html#send_timeout
    send_timeout 120s;
    keepalive_timeout 120s;
    # Cache
    proxy_cache_valid 200 301 302 401 403 404 1d;
    proxy_cache_bypass $http_pragma $no_cache_method $no_cache_control $no_cache_session;
    proxy_cache_revalidate on;
    proxy_cache cache;
    add_header X-Cache-Status $upstream_cache_status;
    # Rate limit
    limit_req zone=bots burst=2 nodelay;
    limit_req zone=one burst=1000 nodelay;
    limit_req_status 429;
    # Proxy
    proxy_set_header X-Forwarded-Proto $web_proxy_scheme;
    proxy_set_header Host $http_host;
    proxy_redirect off;
    proxy_pass http://django_server;
  }

  proxy_intercept_errors on;
  error_page 500 501 502 503 504 505 506 /500.html;

  location = /500.html {
    root /app/500;
    internal;
  }
}