Develop

GetValidator.py

@@ -0,0 +1,12 @@
+from marshmallow import Schema, fields, validates, ValidationError
+from allcities import cities
+
+
+class RunRequestSchema(Schema):
+
+    city = fields.Str(required=True)
+
+    @validates('city')
+    def valid_city(self, value):
+        if len(cities.filter(name=value)) == 0:
+            raise ValidationError("Can't find this city!")

LentaParser.py

@@ -0,0 +1,38 @@
+from bs4 import BeautifulSoup as bs
+import requests
+import time
+from conf import con_lenta
+from ParentNews import ParentNews
+from api_not_available import ApiNotAvailableException
+
+
+class LentaParser(ParentNews):
+
+    def _send_request(self):
+        response = requests.get(f"{con_lenta['url']}")
+        if response.status_code != 200:
+            raise ApiNotAvailableException("Error occurred in LentaParser")
+        else:
+            self.news_data = response.text
+
+    def get_top_news(self):
+        soup = bs(self.news_data, 'lxml')
+        output_news = []
+        for each in soup.select('div[class*="yellow-box__wrap"]'):
+            children = each.findChildren(recursive=False)
+            main_news = children[1:]
+            for news in main_news:
+                news_output = {}
+                news_title = news.getText()
+                news_title = news_title.replace(u'\xa0', u' ')
+                news_output['title'] = news_title
+                news_href = con_lenta['url'] + news.find('a', href=True).get('href')
+                body = requests.get(news_href).text
+                time.sleep(1)
+                soup_article = bs(body, 'lxml')
+                for element in soup_article.select('div[itemprop*="articleBody"]'):
+                    paragraph = element.find_all('p')
+                    all_paragraphs = (''.join(p.getText() for p in paragraph))
+                    news_output['body'] = all_paragraphs
+                output_news.append(news_output)
+        return output_news

ParentNews.py

@@ -0,0 +1,12 @@
+from abc import ABC, abstractmethod
+
+
+class ParentNews(ABC):
+
+    def __init__(self):
+        self.news_data = []
+        self._send_request()
+
+    @abstractmethod
+    def _send_request(self):
+        raise NotImplementedError

ParentWeatherApi.py

@@ -0,0 +1,26 @@
+from abc import ABC, abstractmethod
+
+
+class ParentWeatherApi(ABC):
+
+    def __init__(self, city):
+        self.weather_data = {}
+        self.city = city
+        self._send_request()
+
+    @abstractmethod
+    def _send_request(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_wind(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_weather_description(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_temperature(self):
+        raise NotImplementedError
+

api_not_available.py

@@ -0,0 +1,2 @@
+class ApiNotAvailableException(Exception):
+    pass

cache_data_job.py

@@ -0,0 +1,16 @@
+from db_insert_data import InsertData
+from city_to_country import city_to_country
+
+
+def cache_data_job():
+    cities = InsertData.get_cities()
+    countries = [city_to_country(city) for city in cities]
+    city_ids = [InsertData.check_city_id(city) for city in cities]
+    country_ids = [InsertData.check_country_id(country) for country in countries]
+    for city, city_id in zip(cities, city_ids):
+        InsertData.insert_data_weather_api(city, city_id)
+    for country, country_id in zip(countries, country_ids):
+        InsertData.insert_data_news_api(country, country_id)
+
+
+cache_data_job()

cache_data_job_runner.bat

@@ -0,0 +1 @@
+C:\Users\Admin\anaconda3\python.exe "f:/HDD2/projects_py/get_weather/cache_data_job.py"

city_to_country.py

@@ -0,0 +1,10 @@
+from allcities import cities
+import pycountry
+
+
+def city_to_country(city):
+    filtered_city_set = cities.filter(name=city, population='>500000')
+    largest_city = next(iter(filtered_city_set))
+    if largest_city.dict['country_code'] in [country.alpha_2 for country in list(pycountry.countries)]:
+        country_code = largest_city.dict['country_code']
+        return country_code

conf.py

@@ -1,16 +1,23 @@
 
-
 con = {
-    "api_key": "ca6a19006afff952f0e5246316b1ff94",
+    "api_key": "ca6a19006afff952f0e5246316b1ff944",
     "url": "http://api.weatherstack.com/"
 }
 
-# con_ow_data = {
-#     "api_key": "0a0b534b809a9d3ff21530f757d9509d",
-#     "url": "https://api.openweathermap.org/"
-# }
-
 con_wb = {
     "api_key": "a7e7248f711a4aeea8d6b7529040e94b",
     "url": "https://api.weatherbit.io/v2.0/current"
+}
+
+con_api_news = {
+    "api_key": "0ffe15ac03b847999aa821a7eb904453",
+    "url": "http://newsapi.org/v2/top-headlines"
+}
+
+con_lenta = {
+    "url": "https://lenta.ru/"
+}
+
+postgres_con = {
+    "c_str": "postgresql://postgres:123QWEasd@localhost:5432/api_data_test_2"
 }

db_creation.py

@@ -0,0 +1,48 @@
+from sqlalchemy import create_engine, func, Column, Integer, String, ForeignKey, DateTime
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+Base = declarative_base()
+engine = create_engine("postgresql://postgres:123QWEasd@localhost:5432/api_data_test_2")
+
+
+class Country(Base):
+    __tablename__ = "country"
+
+    id = Column('id', Integer, autoincrement=True, primary_key=True)
+    name = Column('name', String, unique=True)
+    news = relationship("News", back_populates="country")
+
+
+class News(Base):
+    __tablename__ = "news"
+
+    id = Column('id', Integer, autoincrement=True, primary_key=True)
+    country_id = Column(Integer, ForeignKey('country.id'))
+    country = relationship("Country", back_populates="news")
+    title = Column('title', String)
+    body = Column('body', String)
+    date = Column('dateadded', DateTime, default=func.now())
+
+
+class City(Base):
+    __tablename__ = "city"
+
+    id = Column('id', Integer, autoincrement=True, primary_key=True)
+    name = Column('name', String, unique=True)
+    weather = relationship("Weather", back_populates="city")
+
+
+class Weather(Base):
+    __tablename__ = "weather"
+
+    id = Column('id', Integer, autoincrement=True, primary_key=True)
+    city_id = Column(Integer, ForeignKey('city.id'))
+    city = relationship("City", back_populates="weather")
+    weather_info = Column('weather_info', String)
+    temp_in_c = Column('temp_in_c', Integer)
+    wind_speed_kmph = Column('wind_speed_kmph', Integer)
+    date = Column('dateadded', DateTime, default=func.now())
+
+
+Base.metadata.create_all(bind=engine)

db_insert_data.py

@@ -0,0 +1,137 @@
+from conf import postgres_con
+from weather_api_client import WeatherApiClient
+from news_api_client import NewsApiClient
+from weatherbit_api_client import WeatherbitApiClient
+from LentaParser import LentaParser
+from api_not_available import ApiNotAvailableException
+from db_creation import Country, City, News, Weather
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+import time
+
+
+class InsertData:
+    engine = create_engine(postgres_con['c_str'])
+
+    @staticmethod
+    def check_country_id(country_name):
+        session = sessionmaker(bind=InsertData.engine)()
+        country_object = session.query(Country).filter(Country.name == country_name).first()
+        session.close()
+        if country_object:
+            return country_object.id
+        return None
+
+    @staticmethod
+    def check_news(country_id):
+        session = sessionmaker(bind=InsertData.engine)()
+        actual_news = []
+        for instance in session.query(News).filter(News.country_id == country_id).order_by(News.date.desc()).limit(3):
+            actual_news.append({'title': instance.title, 'body': instance.body})
+        session.close()
+        return actual_news
+
+    @staticmethod
+    def insert_country(country_name):
+        session = sessionmaker(bind=InsertData.engine)()
+        country = Country()
+        country.name = country_name
+        session.add(country)
+        session.flush()
+        country_id = country.id
+        session.commit()
+        session.close()
+        return country_id
+
+    @staticmethod
+    def insert_data_news_api(country, country_id):
+        session = sessionmaker(bind=InsertData.engine)()
+        try:
+            whole_news = NewsApiClient(country)
+        except ApiNotAvailableException:
+            time.sleep(1)
+            whole_news = LentaParser()
+        top_news = whole_news.get_top_news()
+        actual_news = []
+        for content in top_news:
+            news = News()
+            news.country_id = country_id
+            news.title = content['title']
+            news.body = content['body']
+            actual_news.append({'title': news.title, 'body': news.body})
+            session.add(news)
+        session.commit()
+        session.close()
+        if actual_news:
+            return actual_news
+        raise RuntimeError('Error during the data processing')
+
+
+    @staticmethod
+    def check_city_id(city_name):
+        session = sessionmaker(bind=InsertData.engine)()
+        city_object = session.query(City).filter(City.name == city_name).first()
+        session.close()
+        if city_object:
+            return city_object.id
+        return None
+
+    @staticmethod
+    def check_weather(city_id):
+        session = sessionmaker(bind=InsertData.engine)()
+        actual_weather = []
+        for instance in session.query(Weather).filter(Weather.city_id == city_id)\
+                                              .order_by(Weather.date.desc())\
+                                              .limit(1):
+            actual_weather.append({
+               'temperature_info': [instance.temp_in_c, 'celsius'],
+               'weather_info': instance.weather_info,
+               'wind_info': [instance.wind_speed_kmph, 'km/h']
+            })
+        session.close()
+        return actual_weather
+
+    @staticmethod
+    def insert_city(city_name):
+        session = sessionmaker(bind=InsertData.engine)()
+        city = City()
+        city.name = city_name
+        session.add(city)
+        session.flush()
+        city_id = city.id
+        session.commit()
+        session.close()
+        return city_id
+
+    @staticmethod
+    def insert_data_weather_api(city, city_id):
+        session = sessionmaker(bind=InsertData.engine)()
+        try:
+            new_weather = WeatherApiClient(city)
+        except ApiNotAvailableException:
+            new_weather = WeatherbitApiClient(city)
+        weather = Weather()
+        weather.city_id = city_id
+        weather.weather_info = new_weather.get_weather_description()
+        weather.temp_in_c = new_weather.get_temperature()[0]
+        weather.wind_speed_kmph = new_weather.get_wind()[0]
+        session.add(weather)
+        actual_weather = {
+            "wind_info": [weather.wind_speed_kmph, 'km/h'],
+            "weather_info": weather.weather_info,
+            "temperature_info": [weather.temp_in_c, "celsius"]
+        }
+        session.commit()
+        session.close()
+        if actual_weather:
+            return actual_weather
+        raise RuntimeError('Error during the data processing')
+
+    @staticmethod
+    def get_cities():
+        actual_city = []
+        session = sessionmaker(bind=InsertData.engine)()
+        city_objects = session.query(City).order_by(City.name).distinct()
+        for city in city_objects:
+            actual_city.append(city.name)
+        return actual_city