switching between tabs fixed, pdf crawling added,scrapy modified

crawling__iitd/crawling__iitd/items.py

@@ -1,12 +1,94 @@
 # Define here the models for your scraped items
-#
 # See documentation in:
 # https://docs.scrapy.org/en/latest/topics/items.html
-
+import sys
+import os
+import itemloaders
 import scrapy
+from elasticsearch import helpers, Elasticsearch
+from elasticsearch.helpers import bulk
+#from elasticsearch_dsl.connections import connections
+from scrapy.loader.processors import MapCompose, TakeFirst, Join
+
+from scrapy.loader import ItemLoader #Import the ItemLoader class and load the items container class to fill the data
+#import ItemLoader.processors.TakeFirst
+
+class CrawlingIitdItemLoader(ItemLoader): #Custom Loader inherits ItemLoader class, call this class on the crawler page to fill the data to Item class
+    default_output_processor = itemloaders.processors.TakeFirst() #The ItemLoader class is used by default to load the items container class to fill the data. It is a list type. You can get the contents of the list through the TakeFirst () method
 
 
+def tianjia(value): #Custom data preprocessing function
+    return value #Return the processed data to the Item. Currently the data processing is being done in spidey.py file so nothing special to write here, just returning the values here.
+
+bulk_lst=[] #for bulk indexing in ES using a list of documnets(i.e. list of Item with some extra fields) <blk_lst>
+host='localhost:9200'
+ELASTIC_INDEX_NAME="iitd_sites"
 class CrawlingIitdItem(scrapy.Item):
     # define the fields for your item here like:
-    # name = scrapy.Field()
-    pass
+    title = scrapy.Field (#Receive title information obtained by the crawler
+        input_processor = itemloaders.processors.MapCompose(tianjia), #Demo of data processing application in fields of Item.Pass the data preprocessing function name into the MapCompose method for processing. The formal parameter value of the data preprocessing function will automatically receive the field title   
+    )
+    url = scrapy.Field ()
+    body = scrapy.Field ()
+    linked_urls = scrapy.Field ()
+    linked_images = scrapy.Field ()
+    meta_tags = scrapy.Field ()
+
+    def save_to_es (self,item): #function for saving the data stored in Item of every crawled webpage to ES
+
+        #if-else coditions are required as some of the crawled urls(webpages) don't have some of the fields in their Item. 
+        response_url = item["url"]
+        if "title" in item.keys(): 
+            response_title=item["title"]
+        else:
+            response_title=item["body"][:10]
+
+        response_body=item["body"]
+
+        if "linked_images" in item.keys():
+            response_images=item["linked_images"]
+        else:
+            response_images={"img":[]}
+
+        if "linked_urls" in item.keys():
+            response_links=item["linked_urls"]
+        else:
+            response_links={}
+
+        if "meta_tags" in item.keys():
+            response_meta=item["meta_tags"]
+        else:
+            response_meta=""
+        #print(item)
+
+        es_client=Elasticsearch(host)
+        if not es_client.indices.exists(ELASTIC_INDEX_NAME):
+            es_client.indices.create(ELASTIC_INDEX_NAME)#initialise index with index_name=elastic_index_name
+
+        es_client.indices.refresh(ELASTIC_INDEX_NAME) #to get the count of documents indexed in "iitd-sites" on ES, refreshing is required.
+        es_len=es_client.cat.count(ELASTIC_INDEX_NAME, params={"format": "json"})
+
+        if int(es_len[0]['count'])>100: #you can define total number of documents you want to index in ES
+            print("100 DOCUMENTS INDEXED--------------","\n")
+            os._exit(0)
+
+        if not es_client.exists(index=ELASTIC_INDEX_NAME, id=response_url):
+            bulk_lst.append({"_index":"iitd_sites",
+                          "_type":"_doc",
+                          "_id":response_url,
+                          "url":response_url,
+                          "title":response_title,
+                          "body":response_body,
+                          "links":response_links,
+                          "linked_images":response_images,
+                          "meta_data":response_meta,
+                          "visits":0
+                        })
+
+        if len(bulk_lst)>=10:
+            bulk(es_client,bulk_lst)
+            bulk_lst.clear()
+            return
+        else:
+            pass  
+

crawling__iitd/crawling__iitd/main.py

@@ -0,0 +1,7 @@
+import sys
+import os
+from scrapy.cmdline import execute #Import and execute scrapy command method
+
+sys.path.append (os.path.join (os.getcwd())) #Add a new path to the Python interpreter, add the directory where the main.py file is located to the Python interpreter
+execute (['scrapy', 'crawl', 'IITD','--nolog']) #execute the scrapy command
+

crawling__iitd/crawling__iitd/pipelines.py

@@ -1,5 +1,4 @@
 # Define your item pipelines here
-#
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 
@@ -8,6 +7,8 @@
 from itemadapter import ItemAdapter
 
 
-class CrawlingIitdPipeline:
+class CrawlingIitdPipeline(object):
     def process_item(self, item, spider):
+        item.save_to_es(item) #Execute the save_to_es method of the items.py file to write data to the elasticsearch search engine
         return item
+

crawling__iitd/crawling__iitd/settings.py

@@ -19,16 +19,13 @@
 # Obey robots.txt rules
 ROBOTSTXT_OBEY = True
 
-#ITEM_PIPELINES={'scrapy.pipelines.images.ImagesPipeline':1}
-#IMAGES_STORE='local_folder'
-
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
 #CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
 # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
+DOWNLOAD_DELAY = 4
 # The download delay setting will honor only one of:
 #CONCURRENT_REQUESTS_PER_DOMAIN = 16
 #CONCURRENT_REQUESTS_PER_IP = 16
@@ -65,22 +62,22 @@
 
 # Configure item pipelines
 # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
-#    'crawling__iitd.pipelines.CrawlingIitdPipeline': 300,
-#}
+ITEM_PIPELINES = {
+    'crawling__iitd.pipelines.CrawlingIitdPipeline': 300,
+}
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
+AUTOTHROTTLE_ENABLED = True
 # The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
+AUTOTHROTTLE_START_DELAY = 5
 # The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
+AUTOTHROTTLE_MAX_DELAY = 60
 # The average number of requests Scrapy should be sending in parallel to
 # each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 # Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
+AUTOTHROTTLE_DEBUG = True
 
 # Enable and configure HTTP caching (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
@@ -89,18 +86,3 @@
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
-FEEDS={
-    'elastic':{
-        'format':'elastic'
-
-    }
-}
-FEED_EXPORTERS = {
-    'elastic': 'crawling__iitd.elastic_exporter.ElasticExporter'
-}
-
-#for crawling in BFO
-DEPTH_PRIORITY = 1
-SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
-SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
-SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.ScrapyPriorityQueue'