Skip to content

Web bugfix #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions src/backend.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
from flask import Flask, render_template, request, url_for, flash, redirect
import datetime
import text_to_emoji
import json
import spotify

app = Flask(__name__) # Initializing flask app


# Route for seeing a data
@app.route("/data")
def get_time():
x = datetime.datetime.now()
# Returning an api for showing in reactjs
return {"🧾💭🤟🧾🍭": "🧾💭🤟🧾🍭", "🧾💭🤟🧾🍭": "🧾💭🤟🧾🍭", "🧾💭🤟🧾🍭": x, "🧾💭🤟🧾🍭": "🧾💭🤟🧾🍭"}


@app.route("/spotify", methods=["GET", "POST"])
def recive_song_name():
song_name = request.form["song_name"]
print(song_name)
return {"data": {"song_name": song_name}}


@app.route("/spotify_emojis")
def get_emojis():
for i in recive_song_name():
song_name = i
emoji = text_to_emoji.text_to_emoji(song_name)
return {"data": {"emoji": emoji}}



# Running app
if __name__ == "__main__":
app.run(debug=True)
18 changes: 11 additions & 7 deletions src/batch_translate_text.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,25 @@ def batch_translate_texts(songs: List[Song]):
"""Returns list of texts translated to emojis."""
res = []
for song in songs:
res.append({'title': song.title, 'text': translate_text(song.text)})
res.append({"title": song.title, "text": translate_text(song.text)})
return res


def load_texts_from_file(filename):
"""Returns list of texts from json file."""
file = Path(filename)
with open(file, 'r') as f:
with open(file, "r") as f:
items = json.load(f)
return [Song(text=song['text'], title=song['title']) for song in items if song.get('text')]
return [
Song(text=song["text"], title=song["title"])
for song in items
if song.get("text")
]


def save_texts_to_file(texts: List[Dict], original_filename: str):
filepath = Path(original_filename.replace('.json', '_translated.json'))
with open(filepath, 'w') as f:
filepath = Path(original_filename.replace(".json", "_translated.json"))
with open(filepath, "w") as f:
json.dump(texts, f)
return filepath

Expand All @@ -58,7 +62,7 @@ def translate_given_json_file(filename):
return filepath


if __name__ == '__main__':
if __name__ == "__main__":
given_file_name = argv[1]
result_path = translate_given_json_file(given_file_name)
print(f'Translated texts saved to: \n\t{result_path}')
print(f"Translated texts saved to: \n\t{result_path}")
24 changes: 12 additions & 12 deletions src/batch_translator.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ def load_raw_songs(filename) -> List[SongNormalised]:
"""Returns list of texts from json file."""
file = Path(filename)
items: List[SongNormalised] = []
with open(file, 'r') as f:
with open(file, "r") as f:
items = json.load(f)
return items

Expand All @@ -25,8 +25,8 @@ def load_raw_songs(filename) -> List[SongNormalised]:
def process_song(song_data: SongNormalised) -> SongTranslated:
# Replace this with your song processing logic
lyrics = song_data["lyrics"]
lyrics = lyrics.split('\n')[0:4]
lyrics = '\n'.join(lyrics)
lyrics = lyrics.split("\n")[0:4]
lyrics = "\n".join(lyrics)
if not DRY_RUN:
translated_lyrics = translate_text(lyrics)
else:
Expand All @@ -53,30 +53,30 @@ def process_songs_multithreaded(songs, num_threads=4):
songs_processed += 1
print(f"Processed {songs_processed}/{total_songs} songs", end="\r")


return songs_translated


def write_translated_songs_to_file(songs: List[SongTranslated], raw_filename: str):
filepath = Path(raw_filename.replace('.json', '_translated.json'))
with open(filepath, 'w') as f:
filepath = Path(raw_filename.replace(".json", "_translated.json"))
with open(filepath, "w") as f:
json.dump(songs, f, indent=4, sort_keys=True)
return filepath


if __name__ == "__main__":
# Replace this with your list of songs
songs_raw = load_raw_songs('data/sample_data/top_300_spotify.json')
songs_raw = load_raw_songs("data/sample_data/top_300_spotify.json")
songs = songs_raw[:]
# Specify the number of threads you want to use
num_threads = 3

translated_songs = process_songs_multithreaded(songs, num_threads)


fp = write_translated_songs_to_file(translated_songs, 'data/sample_data/top_300_spotify.json')
print(f'Translated songs saved to: \n\t{fp}')
fp = write_translated_songs_to_file(
translated_songs, "data/sample_data/top_300_spotify.json"
)
print(f"Translated songs saved to: \n\t{fp}")
for song in translated_songs:
print(song['translated_lyrics'])
print('\n'.join(song['lyrics'].split('\n')[0:4]))
print(song["translated_lyrics"])
print("\n".join(song["lyrics"].split("\n")[0:4]))
print()
60 changes: 32 additions & 28 deletions src/chatgpt/text_to_emoji.py
Original file line number Diff line number Diff line change
@@ -1,59 +1,63 @@
import decouple # pip install python-decouple
import decouple # pip install python-decouple
import requests


API_ENDPOINT = 'https://api.openai.com/v1/chat/completions'
API_KEY = decouple.config('OPENAI_API_KEY')
API_ENDPOINT = "https://api.openai.com/v1/chat/completions"
API_KEY = decouple.config("OPENAI_API_KEY")
headers = {
'Authorization': f'Bearer {API_KEY}',
'Content-Type': 'application/json',
"Authorization": f"Bearer {API_KEY}",
"Content-Type": "application/json",
}


def get_prompt(song):
return (f"<s>[INST] "
f"You're an excellent translator from text to emoji. "
f"You know how to replace word with emoji, keeping the meaning ideally. "
f"Read this text. rgeturn it back, but replace each word with emoji . "
f"Your output should contain emojis only. "
f"Ensure that you have only emojis in your output and don't have any alphabet characters. "
f"Text:\n"
f"{song}"
f"\n"
f"[/INST]")
return (
f"<s>[INST] "
f"You're an excellent translator from text to emoji. "
f"You know how to replace word with emoji, keeping the meaning ideally. "
f"Read this text. rgeturn it back, but replace each word with emoji . "
f"Your output should contain emojis only. "
f"Ensure that you have only emojis in your output and don't have any alphabet characters. "
f"Text:\n"
f"{song}"
f"\n"
f"[/INST]"
)


def translate_text(text):
if not text:
return text
data = {
'model': 'gpt-4',
'messages': [{'role': 'system', 'content': 'You are a helpful assistant.'},
{'role': 'user', 'content': get_prompt(text)}, ],
'temperature': 0.3,
"model": "gpt-4",
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": get_prompt(text)},
],
"temperature": 0.3,
}

response = requests.post(API_ENDPOINT, json=data, headers=headers)

if response.status_code == 200:
result = response.json()
return result['choices'][0]['message']['content']
return result["choices"][0]["message"]["content"]
else:
print(f"Error: {response.status_code} - {response.text}")

return None


if __name__ == '__main__':
print('Usage:')
print('from chatgpt.text_to_emoji import translate_text')
print('translate_text(TXT)')
if __name__ == "__main__":
print("Usage:")
print("from chatgpt.text_to_emoji import translate_text")
print("translate_text(TXT)")
print()
print('Can you guess the song?')
song = '''
print("Can you guess the song?")
song = """
I was five and he was six
We rode on horses made of sticks
He wore black and I wore white
He would always win the fight
'''
print(translate_text(song))
"""
print(translate_text(song))
63 changes: 35 additions & 28 deletions src/embeddings.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,12 @@

from song_types import SongTranslated

openai.api_key = decouple.config('OPENAI_API_KEY')
openai.api_key = decouple.config("OPENAI_API_KEY")


def get_embedding(text):
response = openai.Embedding.create(
input=text,
model="text-embedding-ada-002"
)
return response['data'][0]['embedding']
response = openai.Embedding.create(input=text, model="text-embedding-ada-002")
return response["data"][0]["embedding"]


class SongWithEmbedding(SongTranslated):
Expand All @@ -28,14 +26,16 @@ def load_raw_songs(filename) -> List[SongTranslated]:
"""Returns list of texts from json file."""
file = Path(filename)
items: List[SongTranslated] = []
with open(file, 'r') as f:
with open(file, "r") as f:
items = json.load(f)
return items


def write_songs_with_embeddings_to_file(songs: List[SongWithEmbedding], raw_filename: str):
filepath = Path(raw_filename.replace('.json', '_with_embeddings.json'))
with open(filepath, 'w') as f:
def write_songs_with_embeddings_to_file(
songs: List[SongWithEmbedding], raw_filename: str
):
filepath = Path(raw_filename.replace(".json", "_with_embeddings.json"))
with open(filepath, "w") as f:
json.dump(songs, f, indent=4, sort_keys=True)
return filepath

Expand All @@ -46,50 +46,57 @@ def process_songs(songs, cleaner: Optional[Callable[[str], str]] = None):
songs_with_error = 0
for song in songs:
try:
lyrics = song['lyrics']
lyrics = song["lyrics"]
if cleaner:
lyrics = cleaner(lyrics)
lyrics_embedding = get_embedding(lyrics)
lyrics_translated_embedding = get_embedding(song['translated_lyrics'])
lyrics_translated_embedding = get_embedding(song["translated_lyrics"])
except Exception as e:
songs_with_error += 1
print(f"Error processing song: {e}")
else:
with_embeddings.append(SongWithEmbedding(**song,
lyrics_embedding=lyrics_embedding,
translated_lyrics_embedding=lyrics_translated_embedding))
with_embeddings.append(
SongWithEmbedding(
**song,
lyrics_embedding=lyrics_embedding,
translated_lyrics_embedding=lyrics_translated_embedding,
)
)
songs_processed += 1
print(f"Processed {songs_processed}/{len(songs)} songs", end="\r")
print(f"Processed {songs_processed}/{len(songs)} songs with {songs_with_error} errors")
print(
f"Processed {songs_processed}/{len(songs)} songs with {songs_with_error} errors"
)
return with_embeddings



nlp = spacy.load("en_core_web_sm")


def clean_text(lyric):
doc = nlp(lyric)
pos_tags = ['AUX', 'INTJ', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'X']
pos_tags = ["AUX", "INTJ", "PROPN", "PUNCT", "SCONJ", "SYM", "X"]
words = [token.text for token in doc if token.pos_ not in pos_tags] # filter words
lyric = ' '.join(words).split('\n') # make full string
lyric = " ".join(words).split("\n") # make full string
lyric = [i.strip() for i in lyric if len(i) > 15] # clear small lines
lyric = '\n'.join(lyric).split('\n')[:4] # get the first 4 lines only
lyric = '\n'.join(lyric) # completed string
lyric = "\n".join(lyric).split("\n")[:4] # get the first 4 lines only
lyric = "\n".join(lyric) # completed string

return lyric


if __name__ == "__main__":
# Replace this with your list of songs
songs_raw = load_raw_songs('data/sample_data/top_300_spotify_translated.json')
songs = [s for s in songs_raw if s["lyrics"]] # Filter out songs with no lyrics
songs_raw = load_raw_songs("data/sample_data/top_300_spotify_translated.json")
songs = [s for s in songs_raw if s["lyrics"]] # Filter out songs with no lyrics

converted_songs = process_songs(songs, cleaner=clean_text)

fp = write_songs_with_embeddings_to_file(converted_songs, 'data/sample_data/top_300_spotify.json')
print(f'Translated songs saved to: \n\t{fp}')
fp = write_songs_with_embeddings_to_file(
converted_songs, "data/sample_data/top_300_spotify.json"
)
print(f"Translated songs saved to: \n\t{fp}")
for song in converted_songs:
print(song['translated_lyrics'])
print('\n'.join(song['lyrics'].split('\n')[0:4]))
print(song["translated_lyrics"])
print("\n".join(song["lyrics"].split("\n")[0:4]))
print()

7 changes: 5 additions & 2 deletions src/game_page.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,16 @@ def guess(option: str) -> None:


def centered_title(title):
st.markdown("""
st.markdown(
"""
<style>
.centered-title {
text-align: center;
}
</style>
""", unsafe_allow_html=True)
""",
unsafe_allow_html=True,
)
st.markdown(f"<h1 class='centered-title'>{title}</h1>", unsafe_allow_html=True)


Expand Down
8 changes: 5 additions & 3 deletions src/game_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,11 @@ def get_correct_option_emoji(self) -> str:
return translate_text(self.correct_option)

def __repr__(self) -> str:
return f"words: {self.options} \ncorrect: {self.correct_option} " \
f"\ncorrect songs: {self.correct_songs}\nscore:" \
f" {self.score} \ngame over: {self.game_over}"
return (
f"words: {self.options} \ncorrect: {self.correct_option} "
f"\ncorrect songs: {self.correct_songs}\nscore:"
f" {self.score} \ngame over: {self.game_over}"
)


@st.cache_data
Expand Down
Loading