diff --git a/Crawler/crawling__iitd/crawling__iitd/items.py b/Crawler/crawling__iitd/crawling__iitd/items.py index 5cce158..a00c63c 100644 --- a/Crawler/crawling__iitd/crawling__iitd/items.py +++ b/Crawler/crawling__iitd/crawling__iitd/items.py @@ -88,7 +88,7 @@ def save_to_es (self,item): #function for saving the data stored in Item of ever "visits":0 }) - if len(bulk_lst)>=100: + if len(bulk_lst)>=10: bulk(es_client,bulk_lst) bulk_lst.clear() return diff --git a/README.md b/README.md index 5a951ca..11ace36 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repository contains a search engine for domain "iitd.ac.in" To run - 'docker-compose up --build' Crawler limit - 20000 pages -Elastic Bulk export for every 100 pages +Elastic Bulk export for every 10 pages Functionality enabled to limit the no. of requests made to iitd.ac.in per second @@ -21,9 +21,12 @@ Functionality enabled to limit the no. of requests made to iitd.ac.in per second ##### Nginx configuration - / - passes request to frontend -- /iitd_sites - passes request to Elasticsearch +- /iitd_sites/_search - passes search requests to Elasticsearch ##### Nginx listens on port 7000, to change - - change REACT_APP_ELASTIC_URL in search_frontend/Dockerfile - change port in Nginx service in docker-compose.yml + +#### Note +- To run, it may be needed to change the ending sequence in Crawler/crawling__iitd/entry-point.sh between CRLF, LF (whichever works) \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 387fa9a..03792bc 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -7,38 +7,26 @@ server { listen 80; keepalive_timeout 5; + # this /_stats part is not working + location /iitd_sites/_stats { + proxy_pass http://elasticsearch/iitd_sites/_stats; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } - # Only methods HEAD, POST, GET, CONNECT are allowed - location /iitd_sites { - if ($request_method ~* "PUT") { - return 403; - break; - } - if ($request_method ~* "DELETE") { - return 403; - break; - } - if ($request_method ~* "OPTIONS") { - return 403; - break; - } - if ($request_method ~* "TRACE") { - return 403; - break; - } - if ($request_method ~* "PATCH") { - return 403; - break; - } - if ($request_method ~* "CONNECT") { - return 403; - break; - } - proxy_pass http://elasticsearch/iitd_sites; + location /iitd_sites/_search { + proxy_pass http://elasticsearch/iitd_sites/_search; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $host; proxy_redirect off; } + # location /iitd_sites/{ + # proxy_pass http://elasticsearch/iitd_sites/; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header Host $host; + # proxy_redirect off; + # } location / { proxy_pass http://react_app:3000; diff --git a/search_frontend/src/pages/SearchPage.jsx b/search_frontend/src/pages/SearchPage.jsx index a99662f..8474fc1 100644 --- a/search_frontend/src/pages/SearchPage.jsx +++ b/search_frontend/src/pages/SearchPage.jsx @@ -266,15 +266,16 @@ function SearchPage({query, all="_active",profs="",courses="", images=""}) { function linkClicked(item) { -client.update({ - index: "iitd_sites", - id:item["url"], - body: { - doc:{ - visits:item["visits"]+1 - } - } -}); +// client.update({ +// index: "iitd_sites", +// id:item["url"], +// body: { +// doc:{ +// visits:item["visits"]+1 +// } +// } +// }); + }