diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 8ef4973..2221530 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -43,6 +43,7 @@ jobs:
- name: Integration tests run
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ GENIUS_ACCESS_TOKEN: ${{ secrets.GENIUS_ACCESS_TOKEN }}
run: |
coverage run -m pytest api/integration_tests.py
coverage xml -o coverage.xml
diff --git a/.gitignore b/.gitignore
index 1c2ff08..8b22717 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,4 +30,5 @@ api/__pycache__/
*.pyc
*.tsbuildinfo
.env
-.venv
\ No newline at end of file
+.venv
+api/anki_deck.db
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 297e970..75ef642 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -4,7 +4,10 @@
-
+
+
+
+
@@ -43,27 +46,71 @@
- {
+ "keyToString": {
+ "ModuleVcsDetector.initialDetectionPerformed": "true",
+ "Python tests.Python tests in integration_tests.py.executor": "Run",
+ "Python tests.Python tests in unit_tests1.py.executor": "Run",
+ "Python.Appearance.executor": "Debug",
+ "Python.fast_api.executor": "Run",
+ "RunOnceActivity.ShowReadmeOnStart": "true",
+ "RunOnceActivity.git.unshallow": "true",
+ "git-widget-placeholder": "gen-api",
+ "node.js.detected.package.eslint": "true",
+ "node.js.detected.package.tslint": "true",
+ "node.js.selected.package.eslint": "(autodetect)",
+ "node.js.selected.package.tslint": "(autodetect)",
+ "nodejs_package_manager_path": "npm",
+ "settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable",
+ "ts.external.directory.path": "C:\\Program Files\\JetBrains\\PyCharm 2025.1.1.1\\plugins\\javascript-plugin\\jsLanguageServicesImpl\\external",
+ "vue.rearranger.settings.migration": "true"
}
-}]]>
-
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -82,7 +129,9 @@
+
@@ -106,6 +155,8 @@
+
+
@@ -146,17 +197,17 @@
- file://$PROJECT_DIR$/api/decryptors.py
-
+ file://$PROJECT_DIR$/api/database_preparation.py
+
-
+
-
+
\ No newline at end of file
diff --git a/api/anki_deck.db b/api/anki_deck.db
deleted file mode 100644
index b92946b..0000000
Binary files a/api/anki_deck.db and /dev/null differ
diff --git a/api/database_preparation.py b/api/database_preparation.py
new file mode 100644
index 0000000..a4a32d2
--- /dev/null
+++ b/api/database_preparation.py
@@ -0,0 +1,90 @@
+from sqlite3 import connect
+import os
+
+
+def prepare_db():
+ basedir = os.path.abspath(os.path.dirname(__file__))
+ data_file = os.path.join(basedir, 'anki_deck.db')
+
+ con = connect(data_file)
+ cur = con.cursor()
+
+ cur.execute("""CREATE TABLE user (
+ user_id INTEGER PRIMARY KEY
+ NOT NULL
+ UNIQUE,
+ login TEXT UNIQUE
+ NOT NULL,
+ encrypted_password TEXT NOT NULL
+ DEFAULT (38029384023840 - 2)
+ )""")
+ con.commit()
+
+ cur.execute("""CREATE TABLE words (
+ word_id INTEGER PRIMARY KEY
+ UNIQUE
+ NOT NULL,
+ word TEXT NOT NULL,
+ context_sentence TEXT,
+ user_id INTEGER REFERENCES user (user_id) ON DELETE CASCADE
+ ON UPDATE CASCADE
+ MATCH SIMPLE
+ NOT NULL
+ )""")
+ con.commit()
+
+ cur.execute("""CREATE TABLE cards (
+ card_id INTEGER PRIMARY KEY
+ UNIQUE
+ NOT NULL
+ REFERENCES words (word_id) ON DELETE CASCADE
+ ON UPDATE CASCADE
+ MATCH SIMPLE,
+ word_id INTEGER REFERENCES words (word_id) ON DELETE CASCADE
+ ON UPDATE CASCADE
+ MATCH SIMPLE
+ NOT NULL,
+ sentences BLOB
+ )""")
+ con.commit()
+
+ cur.execute("""CREATE TABLE decks (
+ deck_id INTEGER PRIMARY KEY
+ UNIQUE
+ NOT NULL,
+ cards BLOB,
+ user_id INTEGER REFERENCES user (user_id) ON DELETE CASCADE
+ ON UPDATE CASCADE
+ MATCH SIMPLE
+ NOT NULL
+ )""")
+ con.commit()
+
+ cur.execute("""CREATE TABLE known_words (
+ word_id INTEGER NOT NULL
+ REFERENCES words (word_id) ON DELETE CASCADE
+ ON UPDATE CASCADE
+ MATCH SIMPLE
+ UNIQUE
+ )""")
+ con.commit()
+
+ cur.execute("""CREATE TABLE unknown_words (
+ word_id INTEGER REFERENCES words (word_id) ON DELETE CASCADE
+ ON UPDATE CASCADE
+ MATCH SIMPLE
+ NOT NULL
+ UNIQUE
+ )""")
+ con.commit()
+
+ cur.execute("""CREATE TABLE unwanted_words (
+ word_id INTEGER NOT NULL
+ UNIQUE
+ REFERENCES words (word_id) ON DELETE CASCADE
+ ON UPDATE CASCADE
+ MATCH SIMPLE
+ )""")
+ con.commit()
+
+ con.close()
diff --git a/api/fast_api.py b/api/fast_api.py
index 6650d29..bce6d88 100644
--- a/api/fast_api.py
+++ b/api/fast_api.py
@@ -1,20 +1,21 @@
-import uvicorn
-from fastapi import FastAPI, Query,Request, Response
-from fastapi.responses import JSONResponse, PlainTextResponse
-from sqlite3 import connect
+from fastapi import FastAPI, Query, Response
+from fastapi.responses import PlainTextResponse
from gpt import request_sentences, write_cards_to_csv, parse_response_to_dicts
-import os
-from starlette.responses import JSONResponse
from fastapi.middleware.cors import CORSMiddleware
from decryptors import *
-from pydantic import BaseModel
-from typing import List
+from sqlite3 import connect
+from genius import *
from Appearance import *
-import json
from io import StringIO
import csv
import spacy
from models import *
+from database_preparation import *
+
+if os.path.exists('anki_deck.db'):
+ pass
+else:
+ prepare_db()
app = FastAPI()
basedir = os.path.abspath(os.path.dirname(__file__))
@@ -29,6 +30,7 @@
allow_headers=["*"],
)
+correct_rows = []
@app.get("/")
def root():
@@ -185,38 +187,28 @@ async def post_word(word: str, context_sentence: str, user_id: int, mode: str):
return {"status": 'ok'}
-@app.get("/wordlist/get")
-async def get_wordlist():
+@app.get("/wordlist/get", response_model=WordListGet)
+async def get_wordlist(payload: WordListGet):
con = connect(data_file)
cur = con.cursor()
- ids = cur.execute("SELECT word_id FROM known_words").fetchall()
- words = []
- for word_id in ids:
- word = cur.execute("SELECT word FROM words WHERE word_id = ?", (word_id[0],)).fetchall()
- if word:
- words.append(word[0])
+ words = payload.wordlist
+ ans = {}
+ for word in words:
+ word_id = cur.execute("SELECT word_id FROM words WHERE word = ?", (word,)).fetchone()
+ if word_id:
+ word_id = word_id[0]
+ known_id = cur.execute("SELECT word_id FROM known_words WHERE word_id = ?", (word_id,)).fetchone()
+ if known_id:
+ ans[word] = "known"
+
+ unknown_id = cur.execute("SELECT word_id FROM unknown_words WHERE word_id = ?", (word_id,)).fetchone()
+ if unknown_id:
+ ans[word] = "unknown"
+ else:
+ ans[word] = "none"
con.close()
- return {"words": words}
-
-
-def csv_generation(unknown_words, known_words, count, context_sentences):
- words_to_generate = unknown_words.copy()
- local_correct_rows = []
-
- while words_to_generate:
- response_text = request_sentences(words_to_generate, known_words, count, context_sentences)
- rows = parse_response_to_dicts(response_text)
- still_incorrect = []
- for row in rows:
- word = row["word"]
- sentences = [row["sentence1"]]
- if is_word_in_generated_sentences(word, sentences):
- local_correct_rows.append(row)
- else:
- still_incorrect.append(word)
- words_to_generate = still_incorrect
+ return ans
- return local_correct_rows
@app.post("/wordlist/post", response_model=WordListRequest)
async def post_text(payload: WordListRequest):
@@ -246,18 +238,22 @@ async def post_text(payload: WordListRequest):
con.commit()
ids += 1
con.close()
- global correct_rows
- correct_rows = csv_generation(unknown_words, known_words, count, context_sentences)
+ words_to_generate = unknown_words.copy()
+ while words_to_generate:
+ response_text = request_sentences(words_to_generate, known_words, count, context_sentences)
+ rows = parse_response_to_dicts(response_text)
+ still_incorrect = []
+ for row in rows:
+ word = row["word"]
+ sentences = [row["sentence1"]]
+ if is_word_in_generated_sentences(word, sentences):
+ correct_rows.append(row)
+ else:
+ still_incorrect.append(word)
+ words_to_generate = still_incorrect
return write_cards_to_csv(correct_rows)
-class RegenerationPatchRequest(BaseModel):
- csv_text: str
- marked_words: List[str]
- known_words: List[str]
- count: int
- context_sentences: List[str]
-
@app.post("/regenerate_patch")
async def regenerate_patch(payload: RegenerationPatchRequest):
reader = csv.DictReader(StringIO(payload.csv_text), delimiter=";")
@@ -318,10 +314,7 @@ async def generate_cards_apkg():
headers={"Content-Disposition": "attachment; filename=cards.apkg"}
)
-#@app.post("/fetch-music/post", response_model=GeniusRequest)
-#async def fetch_music(payload: GeniusRequest):
-# artist, song = payload.artist_song.split(' - ')
-# return JSONResponse({"lyrics": get_genius_text(artist, song)})
-
-#if __name__ == "__main__":
-# uvicorn.run(app, host="127.0.0.1", port=8000)
+@app.post("/fetch-music/post", response_model=GeniusRequest)
+async def fetch_music(payload: GeniusRequest):
+ artist, song = payload.query.split(' - ')
+ return PlainTextResponse(get_genius_text(artist, song))
diff --git a/api/genius.py b/api/genius.py
index d8a474d..f18d445 100644
--- a/api/genius.py
+++ b/api/genius.py
@@ -1,14 +1,29 @@
import os
+from requests.exceptions import Timeout
from dotenv import load_dotenv
from lyricsgenius import Genius
+from time import sleep
load_dotenv()
access_token = os.getenv("GENIUS_ACCESS_TOKEN")
genius = Genius(access_token)
+genius._session.proxies = {
+ 'http': os.getenv('HTTP_PROXY'),
+ 'https': os.getenv('HTTPS_PROXY')
+}
genius.remove_section_headers = True
-genius.skip_non_songs = False
+genius.skip_non_songs = True
genius.excluded_terms = ["(Remix)", "(Live)"]
def get_genius_text(artist_name, song_title):
- song = genius.search_song(title=song_title, artist=artist_name)
- return song.lyrics
+ while True:
+ try:
+ song = genius.search_song(title=song_title, artist=artist_name)
+ lyrics = song.lyrics
+ if "Read More" in lyrics:
+ lyrics = lyrics.split("Read More")[1].strip()
+ lyrics = '\n'.join(lyrics.split('\n')[1:])
+ return lyrics
+ except Timeout:
+ sleep(5)
+
diff --git a/api/gpt.py b/api/gpt.py
index e853e51..87498bb 100644
--- a/api/gpt.py
+++ b/api/gpt.py
@@ -1,4 +1,5 @@
-from openai import OpenAI, Timeout
+import openai
+from openai import OpenAI
import io
import spacy
from fastapi.responses import PlainTextResponse
@@ -59,9 +60,10 @@ def request_sentences(unknown_words,known_words,count,context_sentences):#add ar
temperature=0.7
)
return completion.choices[0].message.content
- except Timeout:
+ except openai.APITimeoutError as e:
sleep(5)
+
def parse_response_to_dicts(response_text):
rows = []
for line in response_text.strip().split('\n'):
@@ -150,7 +152,6 @@ def get_word_audio(word):
)
deck.add_note(note)
- # Write media files to temp dir and collect paths
with tempfile.TemporaryDirectory() as tmpdir:
media_paths = []
for fname, fbytes in media_files:
diff --git a/api/integration_tests.py b/api/integration_tests.py
index 82c8379..6b00564 100644
--- a/api/integration_tests.py
+++ b/api/integration_tests.py
@@ -1,5 +1,7 @@
import pytest
from fastapi.testclient import TestClient
+import os
+from api.database_preparation import prepare_db
from api.fast_api import app
client = TestClient(app)
@@ -11,6 +13,10 @@ def setup_and_teardown():
def test_root():
+ #if os.path.exists('anki_deck.db'):
+ # pass
+ #else:
+ # prepare_db()
response = client.get("/")
assert response.status_code == 200
assert response.json() == {"message": "FastAPI is working!"}
@@ -106,23 +112,19 @@ def test_post_word():
assert response.json() == {"status": "ok"}
-#def test_fetch_music():
-# payload = {
-# "artist_song": "Eminem - Rap God"
-# }
-
-# response = client.post("/fetch-music/post", json=payload)
+def test_fetch_music():
+ payload = {
+ "query": "Eminem - Rap God"
+ }
-# assert response.status_code == 200
+ response = client.post("/fetch-music/post", json=payload)
-# response_json = response.json()
-# assert "lyrics" in response_json
-# assert isinstance(response_json["lyrics"], str)
+ assert response.status_code == 200
def test_wordlist_regeneration_invalid_payload():
payload = {
- "known_words": ["слово1"],
+ "known_words": ["word1"],
"currentStuff": {}
}
response = client.post("/regenerate_patch", json=payload)
diff --git a/api/models.py b/api/models.py
index 6e2a834..4c4c7ec 100644
--- a/api/models.py
+++ b/api/models.py
@@ -9,11 +9,17 @@ class WordListRequest(BaseModel):
context_sentences: List[str]
-class WordListRegeneration(BaseModel):
+class RegenerationPatchRequest(BaseModel):
+ csv_text: str
+ marked_words: List[str]
known_words: List[str]
count: int
- currentStuff: Dict[int, Dict[str, str]]
+ context_sentences: List[str]
class GeniusRequest(BaseModel):
- artist_song: str
+ query: str
+
+
+class WordListGet(BaseModel):
+ wordlist: List[str]
diff --git a/docker-compose.yml b/docker-compose.yml
index 92525cb..ce32d77 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,10 +9,10 @@ services:
environment:
- OPENAI_API_KEY=${OPENAI_API_KEY}
- GENIUS_ACCESS_TOKEN=${GENIUS_ACCESS_TOKEN}
- - HTTP_PROXY=user235092:ws16r4@176.101.58.200:2095
- - HTTPS_PROXY=user235092:ws16r4@176.101.58.200:2095
- - http_proxy=user235092:ws16r4@176.101.58.200:2095
- - https_proxy=user235092:ws16r4@176.101.58.200:2095
+ - HTTP_PROXY=http://user235092:ws16r4@176.101.58.200:2095
+ - HTTPS_PROXY=http://user235092:ws16r4@176.101.58.200:2095
+ - http_proxy=http://user235092:ws16r4@176.101.58.200:2095
+ - https_proxy=http://user235092:ws16r4@176.101.58.200:2095
frontend:
build:
diff --git a/test_stuff/src/pages/filterFromText.vue b/test_stuff/src/pages/filterFromText.vue
index 2d47ec9..c5082a9 100644
--- a/test_stuff/src/pages/filterFromText.vue
+++ b/test_stuff/src/pages/filterFromText.vue
@@ -30,7 +30,7 @@ export default {
//after rendering get data from store
this.text = this.textStore.text;
console.log(this.text);
- this.sentences = this.text.split(/[\.!?\n]+/)
+ this.sentences = this.text.split(/[\n]+/)
.filter((sentance) => sentance.length > 0)
.map((sentance) => {
return sentance.trim();
@@ -43,18 +43,13 @@ export default {
},
methods: {
validateWords() {
- let wordsArr = this.text.trim().split(/(\s+)/);
+ let wordsArr = this.text.trim().split(/(\s+|\n)/);
let sentenceIndex = 0;
let prevWord = "";
this.words = wordsArr.map((word) => {
let currentIndex = sentenceIndex;
- let endOfSentance = /[!?\.]+/.exec(word);
- if (endOfSentance != null) {
- sentenceIndex++
- }
- if (word == '\n') {
- if (/[!?\.]+/.exec(prevWord) == null)
- sentenceIndex++;
+ if (word.includes("\n")) {
+ sentenceIndex++;
return {
word: '',
extrChars:['', '\n'],
@@ -65,6 +60,13 @@ export default {
let regex = /[a-zA-Z'`’-]+/;
let newWord = regex.exec(word);
+ if (word.includes("[") || word.includes("]")) {
+ return {
+ word: '',
+ extrChars: ['', ''],
+ class: "default"
+ }
+ }
if (newWord != null) {
return {
word: newWord[0],
diff --git a/test_stuff/src/pages/input.vue b/test_stuff/src/pages/input.vue
index 355c1f0..d2c3ba9 100644
--- a/test_stuff/src/pages/input.vue
+++ b/test_stuff/src/pages/input.vue
@@ -47,11 +47,11 @@ export default {
let words = text.split(/(\s+|\n)/)
.map((word) => {
let currentIndex = sentenceIndex;
- let endOfSentance = endSentenctRegexp.exec(word);
- if (endOfSentance != null) {
- sentenceIndex++;
- }
- if (word == "\n" && endSentenctRegexp.exec(prevWord) == null) {
+ // let endOfSentance = endSentenctRegexp.exec(word);
+ // if (endOfSentance != null) {
+ // sentenceIndex++;
+ // }
+ if (word.trim() == "\n") {
sentenceIndex++;
}
let newWord = regex.exec(word);
@@ -69,7 +69,7 @@ export default {
.filter((word) => word.word.length > 2);
words = this.toSet(words);
const wordCount = words.length;
- let sentences = text.split(/[\.!?\n]+/)
+ let sentences = text.split(/[\n]+/)
.filter((sentance) => sentance.length > 0)
.map((sentance) => {
return sentance.trim();
@@ -80,32 +80,6 @@ export default {
this.textStoreV.setContext(sentences);
router.push({name: "Filter"});
},
- //this function is not need and should be deleted later
- //when all features on this page will be finished
- async fatchdata() {
- this.pickWords();
- const resp = { // add
- unknown_words: ["give","survive","climb"],
- known_words:["dog","cat","emansipation","Russia"],
- count:6,
- context_sentences: ["I will never give up", "He will survive", "This mount is too high to climb"],
- };
- const response = await fetch("http://127.0.0.1:8000/wordlist/post", {
- method: "POST",
- headers: {
- 'Content-Type' : 'application/json'
- },
- body: JSON.stringify(resp),
- })
- const blob = await response.blob();
- const url = window.URL.createObjectURL(blob);
- let a = document.createElement('a');
- a.href = url;
- a.setAttribute("download", "Anki_deck.csv");
- document.body.appendChild(a);
- a.click();
- a.remove();
- },
async handleSubmit() {
if (this.userText) {
this.goToFilter();
@@ -137,13 +111,15 @@ export default {
let words = this.userText.split(/(\s+|\n)/)
.map((word) => {
let currentIndex = sentenceIndex;
- let endOfSentance = endSentenctRegexp.exec(word);
- if (endOfSentance != null) {
- sentenceIndex++;
- }
- if (word == "\n" && endSentenctRegexp.exec(prevWord) == null) {
+ // let endOfSentance = endSentenctRegexp.exec(word);
+ // if (endOfSentance != null) {
+ // sentenceIndex++;
+ // }
+ if (word.includes("\n")) {
sentenceIndex++;
}
+ if (word.includes("["))
+ return {word: ''};
let newWord = regex.exec(word);
prevWord = word
if (newWord != null) {
@@ -171,8 +147,8 @@ export default {
return;
}
- if (wordCount > 500) {
- alert('Пожалуйста, введите не более 500 слов.');
+ if (wordCount > 3000) {
+ alert('Пожалуйста, введите не более 3000 слов.');
return;
}
else {