Nginx redirects to ES for /iitd_sites/_search only

Crawler/crawling__iitd/crawling__iitd/items.py

@@ -88,7 +88,7 @@ def save_to_es (self,item): #function for saving the data stored in Item of ever
                           "visits":0
                         })
 
-        if len(bulk_lst)>=100:
+        if len(bulk_lst)>=10:
             bulk(es_client,bulk_lst)
             bulk_lst.clear()
             return

README.md

@@ -4,7 +4,7 @@ This repository contains a search engine for domain "iitd.ac.in"
 To run - 'docker-compose up --build'
 
 Crawler limit - 20000 pages
-Elastic Bulk export for every 100 pages
+Elastic Bulk export for every 10 pages
 
 Functionality enabled to limit the no. of requests made to iitd.ac.in per second
 
@@ -21,9 +21,12 @@ Functionality enabled to limit the no. of requests made to iitd.ac.in per second
 
 ##### Nginx configuration 
 - / - passes request to frontend
-- /iitd_sites - passes request to Elasticsearch
+- /iitd_sites/_search - passes search requests to Elasticsearch
 
 
 ##### Nginx listens on port 7000, to change - 
 - change REACT_APP_ELASTIC_URL in search_frontend/Dockerfile
 - change port in Nginx service in docker-compose.yml
+
+#### Note
+- To run, it may be needed to change the ending sequence in Crawler/crawling__iitd/entry-point.sh between CRLF, LF (whichever works)

nginx/nginx.conf

@@ -7,38 +7,26 @@ server {
     listen 80;
     keepalive_timeout 5;
 
+    # this /_stats part is not working
+    location /iitd_sites/_stats {
+        proxy_pass http://elasticsearch/iitd_sites/_stats;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
 
-    # Only methods HEAD, POST, GET, CONNECT are allowed
-    location /iitd_sites {
-      if ($request_method ~* "PUT") {
-        return 403;
-        break;
-      }
-      if ($request_method ~* "DELETE") {
-        return 403;
-        break;
-      }
-      if ($request_method ~* "OPTIONS") {
-        return 403;
-        break;
-      }
-      if ($request_method ~* "TRACE") {
-        return 403;
-        break;
-      }
-      if ($request_method ~* "PATCH") {
-        return 403;
-        break;
-      }
-      if ($request_method ~* "CONNECT") {
-        return 403;
-        break;
-      }
-        proxy_pass http://elasticsearch/iitd_sites;
+    location /iitd_sites/_search {
+        proxy_pass http://elasticsearch/iitd_sites/_search;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
         proxy_redirect off;
     }
+    # location /iitd_sites/{
+    #     proxy_pass http://elasticsearch/iitd_sites/;
+    #     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    #     proxy_set_header Host $host;
+    #     proxy_redirect off;
+    # }
 
     location / {
       proxy_pass http://react_app:3000;

search_frontend/src/pages/SearchPage.jsx

@@ -266,15 +266,16 @@ function SearchPage({query, all="_active",profs="",courses="", images=""}) {
 
 function linkClicked(item)
 {
-client.update({
-    index: "iitd_sites",
-    id:item["url"],
-    body: {
-      doc:{
-        visits:item["visits"]+1
-      }
-    }
-});
+// client.update({
+//     index: "iitd_sites",
+//     id:item["url"],
+//     body: {
+//       doc:{
+//         visits:item["visits"]+1
+//       }
+//     }
+// });
+
 }