diff --git a/GetValidator.py b/GetValidator.py new file mode 100644 index 0000000..69d6b6e --- /dev/null +++ b/GetValidator.py @@ -0,0 +1,12 @@ +from marshmallow import Schema, fields, validates, ValidationError +from allcities import cities + + +class RunRequestSchema(Schema): + + city = fields.Str(required=True) + + @validates('city') + def valid_city(self, value): + if len(cities.filter(name=value)) == 0: + raise ValidationError("Can't find this city!") \ No newline at end of file diff --git a/LentaParser.py b/LentaParser.py new file mode 100644 index 0000000..2b8e88b --- /dev/null +++ b/LentaParser.py @@ -0,0 +1,38 @@ +from bs4 import BeautifulSoup as bs +import requests +import time +from conf import con_lenta +from ParentNews import ParentNews +from api_not_available import ApiNotAvailableException + + +class LentaParser(ParentNews): + + def _send_request(self): + response = requests.get(f"{con_lenta['url']}") + if response.status_code != 200: + raise ApiNotAvailableException("Error occurred in LentaParser") + else: + self.news_data = response.text + + def get_top_news(self): + soup = bs(self.news_data, 'lxml') + output_news = [] + for each in soup.select('div[class*="yellow-box__wrap"]'): + children = each.findChildren(recursive=False) + main_news = children[1:] + for news in main_news: + news_output = {} + news_title = news.getText() + news_title = news_title.replace(u'\xa0', u' ') + news_output['title'] = news_title + news_href = con_lenta['url'] + news.find('a', href=True).get('href') + body = requests.get(news_href).text + time.sleep(1) + soup_article = bs(body, 'lxml') + for element in soup_article.select('div[itemprop*="articleBody"]'): + paragraph = element.find_all('p') + all_paragraphs = (''.join(p.getText() for p in paragraph)) + news_output['body'] = all_paragraphs + output_news.append(news_output) + return output_news diff --git a/ParentNews.py b/ParentNews.py new file mode 100644 index 0000000..e07bfeb --- /dev/null +++ b/ParentNews.py @@ -0,0 +1,12 @@ +from abc import ABC, abstractmethod + + +class ParentNews(ABC): + + def __init__(self): + self.news_data = [] + self._send_request() + + @abstractmethod + def _send_request(self): + raise NotImplementedError diff --git a/ParentWeatherApi.py b/ParentWeatherApi.py new file mode 100644 index 0000000..cd2d319 --- /dev/null +++ b/ParentWeatherApi.py @@ -0,0 +1,26 @@ +from abc import ABC, abstractmethod + + +class ParentWeatherApi(ABC): + + def __init__(self, city): + self.weather_data = {} + self.city = city + self._send_request() + + @abstractmethod + def _send_request(self): + raise NotImplementedError + + @abstractmethod + def get_wind(self): + raise NotImplementedError + + @abstractmethod + def get_weather_description(self): + raise NotImplementedError + + @abstractmethod + def get_temperature(self): + raise NotImplementedError + diff --git a/api_not_available.py b/api_not_available.py new file mode 100644 index 0000000..5d10b1f --- /dev/null +++ b/api_not_available.py @@ -0,0 +1,2 @@ +class ApiNotAvailableException(Exception): + pass diff --git a/cache_data_job.py b/cache_data_job.py new file mode 100644 index 0000000..6796081 --- /dev/null +++ b/cache_data_job.py @@ -0,0 +1,16 @@ +from db_insert_data import InsertData +from city_to_country import city_to_country + + +def cache_data_job(): + cities = InsertData.get_cities() + countries = [city_to_country(city) for city in cities] + city_ids = [InsertData.check_city_id(city) for city in cities] + country_ids = [InsertData.check_country_id(country) for country in countries] + for city, city_id in zip(cities, city_ids): + InsertData.insert_data_weather_api(city, city_id) + for country, country_id in zip(countries, country_ids): + InsertData.insert_data_news_api(country, country_id) + + +cache_data_job() diff --git a/cache_data_job_runner.bat b/cache_data_job_runner.bat new file mode 100644 index 0000000..0614793 --- /dev/null +++ b/cache_data_job_runner.bat @@ -0,0 +1 @@ +C:\Users\Admin\anaconda3\python.exe "f:/HDD2/projects_py/get_weather/cache_data_job.py" \ No newline at end of file diff --git a/city_to_country.py b/city_to_country.py new file mode 100644 index 0000000..cabe7ac --- /dev/null +++ b/city_to_country.py @@ -0,0 +1,10 @@ +from allcities import cities +import pycountry + + +def city_to_country(city): + filtered_city_set = cities.filter(name=city, population='>500000') + largest_city = next(iter(filtered_city_set)) + if largest_city.dict['country_code'] in [country.alpha_2 for country in list(pycountry.countries)]: + country_code = largest_city.dict['country_code'] + return country_code diff --git a/conf.py b/conf.py index b486b98..aa5d8b1 100644 --- a/conf.py +++ b/conf.py @@ -1,16 +1,23 @@ - con = { - "api_key": "ca6a19006afff952f0e5246316b1ff94", + "api_key": "ca6a19006afff952f0e5246316b1ff944", "url": "http://api.weatherstack.com/" } -# con_ow_data = { -# "api_key": "0a0b534b809a9d3ff21530f757d9509d", -# "url": "https://api.openweathermap.org/" -# } - con_wb = { "api_key": "a7e7248f711a4aeea8d6b7529040e94b", "url": "https://api.weatherbit.io/v2.0/current" +} + +con_api_news = { + "api_key": "0ffe15ac03b847999aa821a7eb904453", + "url": "http://newsapi.org/v2/top-headlines" +} + +con_lenta = { + "url": "https://lenta.ru/" +} + +postgres_con = { + "c_str": "postgresql://postgres:123QWEasd@localhost:5432/api_data_test_2" } \ No newline at end of file diff --git a/db_creation.py b/db_creation.py new file mode 100644 index 0000000..75ef644 --- /dev/null +++ b/db_creation.py @@ -0,0 +1,48 @@ +from sqlalchemy import create_engine, func, Column, Integer, String, ForeignKey, DateTime +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +Base = declarative_base() +engine = create_engine("postgresql://postgres:123QWEasd@localhost:5432/api_data_test_2") + + +class Country(Base): + __tablename__ = "country" + + id = Column('id', Integer, autoincrement=True, primary_key=True) + name = Column('name', String, unique=True) + news = relationship("News", back_populates="country") + + +class News(Base): + __tablename__ = "news" + + id = Column('id', Integer, autoincrement=True, primary_key=True) + country_id = Column(Integer, ForeignKey('country.id')) + country = relationship("Country", back_populates="news") + title = Column('title', String) + body = Column('body', String) + date = Column('dateadded', DateTime, default=func.now()) + + +class City(Base): + __tablename__ = "city" + + id = Column('id', Integer, autoincrement=True, primary_key=True) + name = Column('name', String, unique=True) + weather = relationship("Weather", back_populates="city") + + +class Weather(Base): + __tablename__ = "weather" + + id = Column('id', Integer, autoincrement=True, primary_key=True) + city_id = Column(Integer, ForeignKey('city.id')) + city = relationship("City", back_populates="weather") + weather_info = Column('weather_info', String) + temp_in_c = Column('temp_in_c', Integer) + wind_speed_kmph = Column('wind_speed_kmph', Integer) + date = Column('dateadded', DateTime, default=func.now()) + + +Base.metadata.create_all(bind=engine) diff --git a/db_insert_data.py b/db_insert_data.py new file mode 100644 index 0000000..1c6a64a --- /dev/null +++ b/db_insert_data.py @@ -0,0 +1,137 @@ +from conf import postgres_con +from weather_api_client import WeatherApiClient +from news_api_client import NewsApiClient +from weatherbit_api_client import WeatherbitApiClient +from LentaParser import LentaParser +from api_not_available import ApiNotAvailableException +from db_creation import Country, City, News, Weather +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +import time + + +class InsertData: + engine = create_engine(postgres_con['c_str']) + + @staticmethod + def check_country_id(country_name): + session = sessionmaker(bind=InsertData.engine)() + country_object = session.query(Country).filter(Country.name == country_name).first() + session.close() + if country_object: + return country_object.id + return None + + @staticmethod + def check_news(country_id): + session = sessionmaker(bind=InsertData.engine)() + actual_news = [] + for instance in session.query(News).filter(News.country_id == country_id).order_by(News.date.desc()).limit(3): + actual_news.append({'title': instance.title, 'body': instance.body}) + session.close() + return actual_news + + @staticmethod + def insert_country(country_name): + session = sessionmaker(bind=InsertData.engine)() + country = Country() + country.name = country_name + session.add(country) + session.flush() + country_id = country.id + session.commit() + session.close() + return country_id + + @staticmethod + def insert_data_news_api(country, country_id): + session = sessionmaker(bind=InsertData.engine)() + try: + whole_news = NewsApiClient(country) + except ApiNotAvailableException: + time.sleep(1) + whole_news = LentaParser() + top_news = whole_news.get_top_news() + actual_news = [] + for content in top_news: + news = News() + news.country_id = country_id + news.title = content['title'] + news.body = content['body'] + actual_news.append({'title': news.title, 'body': news.body}) + session.add(news) + session.commit() + session.close() + if actual_news: + return actual_news + raise RuntimeError('Error during the data processing') + + + @staticmethod + def check_city_id(city_name): + session = sessionmaker(bind=InsertData.engine)() + city_object = session.query(City).filter(City.name == city_name).first() + session.close() + if city_object: + return city_object.id + return None + + @staticmethod + def check_weather(city_id): + session = sessionmaker(bind=InsertData.engine)() + actual_weather = [] + for instance in session.query(Weather).filter(Weather.city_id == city_id)\ + .order_by(Weather.date.desc())\ + .limit(1): + actual_weather.append({ + 'temperature_info': [instance.temp_in_c, 'celsius'], + 'weather_info': instance.weather_info, + 'wind_info': [instance.wind_speed_kmph, 'km/h'] + }) + session.close() + return actual_weather + + @staticmethod + def insert_city(city_name): + session = sessionmaker(bind=InsertData.engine)() + city = City() + city.name = city_name + session.add(city) + session.flush() + city_id = city.id + session.commit() + session.close() + return city_id + + @staticmethod + def insert_data_weather_api(city, city_id): + session = sessionmaker(bind=InsertData.engine)() + try: + new_weather = WeatherApiClient(city) + except ApiNotAvailableException: + new_weather = WeatherbitApiClient(city) + weather = Weather() + weather.city_id = city_id + weather.weather_info = new_weather.get_weather_description() + weather.temp_in_c = new_weather.get_temperature()[0] + weather.wind_speed_kmph = new_weather.get_wind()[0] + session.add(weather) + actual_weather = { + "wind_info": [weather.wind_speed_kmph, 'km/h'], + "weather_info": weather.weather_info, + "temperature_info": [weather.temp_in_c, "celsius"] + } + session.commit() + session.close() + if actual_weather: + return actual_weather + raise RuntimeError('Error during the data processing') + + @staticmethod + def get_cities(): + actual_city = [] + session = sessionmaker(bind=InsertData.engine)() + city_objects = session.query(City).order_by(City.name).distinct() + for city in city_objects: + actual_city.append(city.name) + return actual_city diff --git a/index.py b/index.py index 07bc85a..f4296d5 100644 --- a/index.py +++ b/index.py @@ -1,80 +1,42 @@ -from weather_api_client import WeatherApiClient -# from open_weather_api_client import OpenWeatherApiClient -from weatherbit_api_client import WeatherbitApiClient -from flask import Flask -from flask import request -from flask import render_template -from flask import redirect -from flask_restful import Resource, Api -from flask import jsonify +from flask import Flask, request, jsonify +from GetValidator import RunRequestSchema +from flask_restful import Api +from city_to_country import city_to_country +from db_insert_data import InsertData app = Flask(__name__) api = Api(app) +run_request_schema = RunRequestSchema() -@app.route('/') -def index(): - return render_template('form.html') - -@app.route('/main', methods=['POST', 'GET']) -def main(): - # city = request.args.get('city') - body = request.get_json(silent=True) - # print(body) - city = body['city'] - if city: - new_weather = WeatherApiClient(city) - status = new_weather.get_data() - if status: - wind_info = new_weather.get_wind() - moisture_info = new_weather.get_moisture() - main_weather_params = new_weather.get_main_weather_params() - data = {"moisture_info": moisture_info, - "wind_info": wind_info, - "main_weather_params": main_weather_params} - return jsonify(data) - else: - # new_weather = OpenWeatherApiClient(city) - new_weather = WeatherbitApiClient(city) - status = new_weather.get_data() - print("status", status) - if status: - weather_descr = new_weather.get_weather() - wind_info = new_weather.get_wind() - main_info = new_weather.get_main() - data = {"weather_descr": weather_descr, - "wind_info": wind_info, - "main_info": main_info} - return jsonify(data) - else: - return render_template('error.html', city=city) - - -@app.route('/wind-info') -def wind(): +@app.route('/current_data', methods=['GET']) +def current_data(): + errors = run_request_schema.validate(request.args) + if errors: + raise ValueError(("An error occurred with input: {}".format(errors))) city = request.args.get('city') - if city: - new_weather = WeatherApiClient(city) - wind_info = new_weather.get_wind() - print(wind_info) - return render_template('wind-info.html', wind=wind_info) + country = city_to_country(city) + country_id = InsertData.check_country_id(country) + city_id = InsertData.check_city_id(city) + if not country_id and not city_id: + country_id = InsertData.insert_country(country) + print(country_id) + city_id = InsertData.insert_city(city) + actual_weather = InsertData.insert_data_weather_api(city, city_id) + actual_news = InsertData.insert_data_news_api(country, country_id) else: - return redirect("/", code=302) - - -@app.route('/moisture-info') -def moisture(): - city = request.args.get('city') - if city: - new_weather = WeatherApiClient(city) - moisture_info = new_weather.get_moisture() - return render_template('moisture-info.html', moisture=moisture_info) - else: - return redirect("/", code=302) - - + actual_weather = InsertData.check_weather(city_id) + actual_news = InsertData.check_news(country_id) + print(actual_news) + data = { + "city": city, + "country": country, + "hot_news": actual_news, + "local_weather": actual_weather + } + return jsonify(data) if __name__ == '__main__': diff --git a/news_api_client.py b/news_api_client.py new file mode 100644 index 0000000..5062039 --- /dev/null +++ b/news_api_client.py @@ -0,0 +1,30 @@ +import requests +from conf import con_api_news +from api_not_available import ApiNotAvailableException +from ParentNews import ParentNews + + +class NewsApiClient(ParentNews): + + def __init__(self, country): + self.country = country + super().__init__() + + def _send_request(self): + response = requests.get(f"{con_api_news['url']}?country={self.country}&apiKey={con_api_news['api_key']}") + if response.status_code != 200: + error_message = response.json()['message'] + raise ApiNotAvailableException(f"{error_message} occurred in NewsApiClient") + else: + self.news_data = response.json() + + def get_top_news(self): + output_news = [] + for news in self.news_data['articles'][0:3]: + if news['title'] and news['content']: + formatted_news = { + "title": news['title'], + "body": news['content'] + } + output_news.append(formatted_news) + return output_news diff --git a/open_weather_api_client.py b/open_weather_api_client.py deleted file mode 100644 index ee0aed2..0000000 --- a/open_weather_api_client.py +++ /dev/null @@ -1,37 +0,0 @@ -import requests -import json -import conf - - -class OpenWeatherApiClient: - - weather_data = {} - - def __init__(self, city): - config = conf.con_ow_data - # self.weather_data = requests.get(config['url'] + "current?access_key=" + config['api_key'] + "&query=" + city) - response = requests.get(f"{config['url']}data/2.5/find?q={city}&appid={config['api_key']}") - self.weather_data = json.loads(response.text) - print(self.weather_data) - - def get_weather(self): - weather_descr = { - "weather_descr": self.weather_data['list'][0]['weather'][0]['description'] - } - return weather_descr - - def get_wind(self): - wind_info = { - "wind_speed": self.weather_data['list'][0]['wind']['speed'], - "wind_degree": self.weather_data['list'][0]['wind']['deg'] - } - return wind_info - - def get_main(self): - main_info = { - "temp": self.weather_data['list'][0]['main']['temp'], - "feels_like": self.weather_data['list'][0]['main']['feels_like'], - "pressure": self.weather_data['list'][0]['main']['pressure'], - "humidity": self.weather_data['list'][0]['main']['humidity'] - } - return main_info diff --git a/pgconnection.py b/pgconnection.py new file mode 100644 index 0000000..ef10058 --- /dev/null +++ b/pgconnection.py @@ -0,0 +1,8 @@ +import psycopg2 + + +def get_connection(dbname): + + connect_str = "dbname={} host='localhost' user='postgres' password='123QWEasd'".format(dbname) + + return psycopg2.connect(connect_str) diff --git a/weather_api_client.py b/weather_api_client.py index 83845e5..9207d55 100644 --- a/weather_api_client.py +++ b/weather_api_client.py @@ -1,46 +1,25 @@ import requests -import conf +from conf import con +from ParentWeatherApi import ParentWeatherApi +from api_not_available import ApiNotAvailableException -class WeatherApiClient: +class WeatherApiClient(ParentWeatherApi): + BASE_URL = "{}current?access_key={}&query={}" - def __init__(self, city): - self.config = conf.con - self.city = city - # self.weather_data = requests.get(config['url'] + "current?access_key=" + config['api_key'] + "&query=" + city) - self.weather_data = {} - - def get_data(self): - response = requests.get(f"{self.config['url']}current?access_key={self.config['api_key']}&query={self.city}") - if 'request' not in response.json(): - return False + def _send_request(self): + response = requests.get(self.BASE_URL.format(con['url'], con['api_key'], self.city)) + if 'success' in response.json(): + error_message = response.json()['error']['info'] + raise ApiNotAvailableException(f"{error_message} occurred in WeatherApiClient") else: self.weather_data = response.json() - print(self.weather_data) - return True def get_wind(self): - wind_info = { - "wind_speed": self.weather_data['current']['wind_speed'], - "wind_degree": self.weather_data['current']['wind_degree'], - "wind_dir": self.weather_data['current']['wind_dir'], - } - return wind_info + return self.weather_data['current']['wind_speed'], "km/h" - def get_moisture(self): - moisture_info = { - "precip": self.weather_data['current']['precip'], - "humidity": self.weather_data['current']['humidity'], - "cloudcover": self.weather_data['current']['cloudcover'], - } - return moisture_info + def get_weather_description(self): + return self.weather_data['current']['weather_descriptions'][0] - def get_main_weather_params(self): - other_info = { - "temperature": self.weather_data['current']['temperature'], - "feelslike": self.weather_data['current']['feelslike'], - "weather_descriptions": self.weather_data['current']['weather_descriptions'][0], - "uv_index": self.weather_data['current']['uv_index'], - "visibility": self.weather_data['current']['visibility'], - } - return other_info + def get_temperature(self): + return self.weather_data['current']['temperature'], "celsius" diff --git a/weatherbit_api_client.py b/weatherbit_api_client.py index d1e26f5..ec3f7b8 100644 --- a/weatherbit_api_client.py +++ b/weatherbit_api_client.py @@ -1,42 +1,29 @@ import requests -import conf +from conf import con_wb +from api_not_available import ApiNotAvailableException +from ParentWeatherApi import ParentWeatherApi -class WeatherbitApiClient: +class WeatherbitApiClient(ParentWeatherApi): + BASE_URL = "{}?city={}&key={}" - def __init__(self, city): - self.config = conf.con_wb - self.city = city - self.weather_data = {} - # self.weather_data = requests.get(config['url'] + "current?access_key=" + config['api_key'] + "&query=" + city) - - def get_data(self): - response = requests.get(f"{self.config['url']}?city={self.city}&key={self.config['api_key']}") - if 'data' not in response.json(): - return False + def _send_request(self): + response = requests.get(self.BASE_URL.format(con_wb['url'], self.city, con_wb['api_key'])) + if response.status_code != 200: + print(self.city) + print(response.text) + print(response.status_code) + error_message = response.json()['error'] + raise ApiNotAvailableException(f"{error_message} occurred in WeatherbitApiClient") else: self.weather_data = response.json() - print(self.weather_data) - return True - - def get_weather(self): - weather_descr = { - "weather_descr": self.weather_data['data'][0]['weather']['description'] - } - return weather_descr + return self.weather_data def get_wind(self): - wind_info = { - "wind_speed": self.weather_data['data'][0]['wind_spd'], - "wind_dir": self.weather_data['data'][0]['wind_cdir_full'] - } - return wind_info + return self.weather_data['data'][0]['wind_spd'], "km/h" + + def get_weather_description(self): + return self.weather_data['data'][0]['weather']['description'] - def get_main(self): - main_info = { - "temp": self.weather_data['data'][0]['temp'], - "feels_like": self.weather_data['data'][0]['app_temp'], - "pressure": self.weather_data['data'][0]['pres'], - "humidity": self.weather_data['data'][0]['rh'] - } - return main_info + def get_temperature(self): + return self.weather_data['data'][0]['temp'], "celsius"