From 105f565540a0ee86242a3c49d5f87f007739a02a Mon Sep 17 00:00:00 2001 From: herz Date: Thu, 17 Mar 2022 11:54:37 +0100 Subject: [PATCH] Made everything parallel --- faceitstalker_docker/app/main.py | 106 ++++++--------- faceitstalker_docker/app/parallel.py | 122 ++++++++++++++++++ .../app/templates/steamids.html | 32 ++--- wip.py | 46 +++++-- 4 files changed, 213 insertions(+), 93 deletions(-) create mode 100644 faceitstalker_docker/app/parallel.py diff --git a/faceitstalker_docker/app/main.py b/faceitstalker_docker/app/main.py index aca7660..6faf68a 100644 --- a/faceitstalker_docker/app/main.py +++ b/faceitstalker_docker/app/main.py @@ -5,8 +5,8 @@ import re import json import xml import xmltodict -import asyncio - +from joblib import Parallel, delayed +import multiprocessing app = Flask(__name__) @@ -14,7 +14,7 @@ app = Flask(__name__) @app.route("/", methods=["POST", "GET"]) -async def index(): +def index(): data_dict = {} if request.method == "POST": @@ -22,33 +22,19 @@ async def index(): steamids = regex_for_steamids(status_screen) + num_cores = multiprocessing.cpu_count() - for steamid in steamids: - data_dict[steamid] = {} - data_dict[steamid]['steamid64'] = steamid_to_64bit(steamid) + data_dict = {} + #steamid64_dict = Parallel(n_jobs=num_cores, verbose=50)(delayed(steamid_to_64bit)(steamid)for steamid in steamids) + data_dict_list = Parallel(n_jobs=num_cores, verbose=50)(delayed(get_evererything)(steamid)for steamid in steamids) - task_get_faceit_data = asyncio.create_task(loop_faceit_data(data_dict)) - task_get_steam_data = asyncio.create_task(loop_steam_data(data_dict)) + for dict in data_dict_list: + for k, v in dict.items(): + data_dict[k] = v + - faceit_dict = await task_get_faceit_data - steam_dict = await task_get_steam_data - - for steamid in faceit_dict: - if faceit_dict[steamid]['faceit_acc'] == 'true' : - data_dict[steamid]['faceit_elo'] = faceit_dict[steamid]['faceit_elo'] - data_dict[steamid]['faceit_level'] = faceit_dict[steamid]['faceit_level'] - data_dict[steamid]['faceit_name'] = faceit_dict[steamid]['faceit_name'] - data_dict[steamid]['faceit_acc'] = faceit_dict[steamid]['faceit_acc'] - else: - data_dict[steamid]['faceit_acc'] = faceit_dict[steamid]['faceit_acc'] - - for steamid in steam_dict: - data_dict[steamid]['steam_name'] = steam_dict[steamid]['steam_name'] - data_dict[steamid]['steam_pic'] = steam_dict[steamid]['steam_pic'] - - - return render_template('steamids.html', data_dict = data_dict , steam_dict = steam_dict, faceit_dict = faceit_dict) + return render_template('steamids.html', data_dict = data_dict) else: return render_template('index.html') @@ -63,53 +49,43 @@ def regex_for_steamids(input_string): steamids_from_status = re.findall(r"\S*STEAM_+[0-9]+:[0-9]+:[0-9]+", input_string) return steamids_from_status -def steamid_to_64bit(steamid): + + +def get_evererything(steamid): + data_dict_funct = {} + data_dict_funct[steamid] = {} + + #Get Steamid64 steam64id = 76561197960265728 # Who tf knows how this works # IT JUST FUCKING DOES! id_split = steamid.split(":") steam64id += int(id_split[2]) * 2 # JUST MULTIPLY BY 2... Cause THAT MAKES PERFEKT SENSE RIGHT?! if id_split[1] == "1": steam64id += 1 - return steam64id - - -async def loop_faceit_data(data_dict): - faceit_dict = {} - req =requests.Session() - for steamid in data_dict: - faceit_dict[steamid] = {} - url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(data_dict[steamid]['steamid64']) - headers = { 'accept': 'application/json', 'Authorization' : 'Bearer ab46a7ab-6ab8-4c00-a8ff-41c0ff71d562' } - faceit_response = req.get(url, headers=headers) - faceit_response_json = faceit_response.json() - - - try: - faceit_dict[steamid]['faceit_elo'] = faceit_response_json['games']['csgo']['faceit_elo'] - faceit_dict[steamid]['faceit_level'] = faceit_response_json['games']['csgo']['skill_level'] - faceit_dict[steamid]['faceit_name'] = faceit_response_json['nickname'] - faceit_dict[steamid]['faceit_acc'] = 'true' - except: - faceit_dict[steamid]['faceit_acc'] = 'false' - - return faceit_dict - -async def loop_steam_data(data_dict): - steam_dict = {} - req =requests.Session() - for steamid in data_dict: - steam_dict[steamid] = {} - steamurl ="https://steamcommunity.com/profiles/" + str(data_dict[steamid]['steamid64']) +"/?xml=1" - steam_req = req.get(steamurl) - steamdict_funct = xmltodict.parse(steam_req.content) - - steam_dict[steamid]['steam_name'] = steamdict_funct['profile']['steamID'] - steam_dict[steamid]['steam_pic'] = steamdict_funct['profile']['avatarFull'] - - return steam_dict - + data_dict_funct[steamid]['steamid64'] = steam64id + + #Get Faceit Data + url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(data_dict_funct[steamid]['steamid64']) + headers = { 'accept': 'application/json', 'Authorization' : 'Bearer ab46a7ab-6ab8-4c00-a8ff-41c0ff71d562' } + faceit_response = requests.get(url, headers=headers) + faceit_response_json = faceit_response.json() + try: + data_dict_funct[steamid]['faceit_elo'] = faceit_response_json['games']['csgo']['faceit_elo'] + data_dict_funct[steamid]['faceit_level'] = faceit_response_json['games']['csgo']['skill_level'] + data_dict_funct[steamid]['faceit_name'] = faceit_response_json['nickname'] + data_dict_funct[steamid]['faceit_acc'] = 'true' + except: + data_dict_funct[steamid]['faceit_acc'] = 'false' + + #Get Steam Data + steamurl ="https://steamcommunity.com/profiles/" + str(data_dict_funct[steamid]['steamid64']) +"/?xml=1" + steam_req = requests.get(steamurl) + steamdict_funct = xmltodict.parse(steam_req.content) + data_dict_funct[steamid]['steam_name'] = steamdict_funct['profile']['steamID'] + data_dict_funct[steamid]['steam_pic'] = steamdict_funct['profile']['avatarFull'] + return data_dict_funct diff --git a/faceitstalker_docker/app/parallel.py b/faceitstalker_docker/app/parallel.py new file mode 100644 index 0000000..6faf68a --- /dev/null +++ b/faceitstalker_docker/app/parallel.py @@ -0,0 +1,122 @@ +from flask import Flask, render_template, request, redirect, url_for +from waitress import serve +import requests +import re +import json +import xml +import xmltodict +from joblib import Parallel, delayed +import multiprocessing + +app = Flask(__name__) + + + + +@app.route("/", methods=["POST", "GET"]) +def index(): + data_dict = {} + + if request.method == "POST": + status_screen = request.form["status_screen_input"] + steamids = regex_for_steamids(status_screen) + + + num_cores = multiprocessing.cpu_count() + + data_dict = {} + #steamid64_dict = Parallel(n_jobs=num_cores, verbose=50)(delayed(steamid_to_64bit)(steamid)for steamid in steamids) + data_dict_list = Parallel(n_jobs=num_cores, verbose=50)(delayed(get_evererything)(steamid)for steamid in steamids) + + for dict in data_dict_list: + for k, v in dict.items(): + data_dict[k] = v + + + + return render_template('steamids.html', data_dict = data_dict) + else: + return render_template('index.html') + + + + + + + + +def regex_for_steamids(input_string): + steamids_from_status = re.findall(r"\S*STEAM_+[0-9]+:[0-9]+:[0-9]+", input_string) + return steamids_from_status + + + +def get_evererything(steamid): + data_dict_funct = {} + data_dict_funct[steamid] = {} + + #Get Steamid64 + steam64id = 76561197960265728 # Who tf knows how this works + # IT JUST FUCKING DOES! + id_split = steamid.split(":") + steam64id += int(id_split[2]) * 2 # JUST MULTIPLY BY 2... Cause THAT MAKES PERFEKT SENSE RIGHT?! + if id_split[1] == "1": + steam64id += 1 + data_dict_funct[steamid]['steamid64'] = steam64id + + #Get Faceit Data + url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(data_dict_funct[steamid]['steamid64']) + headers = { 'accept': 'application/json', 'Authorization' : 'Bearer ab46a7ab-6ab8-4c00-a8ff-41c0ff71d562' } + faceit_response = requests.get(url, headers=headers) + faceit_response_json = faceit_response.json() + try: + data_dict_funct[steamid]['faceit_elo'] = faceit_response_json['games']['csgo']['faceit_elo'] + data_dict_funct[steamid]['faceit_level'] = faceit_response_json['games']['csgo']['skill_level'] + data_dict_funct[steamid]['faceit_name'] = faceit_response_json['nickname'] + data_dict_funct[steamid]['faceit_acc'] = 'true' + except: + data_dict_funct[steamid]['faceit_acc'] = 'false' + + #Get Steam Data + steamurl ="https://steamcommunity.com/profiles/" + str(data_dict_funct[steamid]['steamid64']) +"/?xml=1" + steam_req = requests.get(steamurl) + steamdict_funct = xmltodict.parse(steam_req.content) + data_dict_funct[steamid]['steam_name'] = steamdict_funct['profile']['steamID'] + data_dict_funct[steamid]['steam_pic'] = steamdict_funct['profile']['avatarFull'] + + + return data_dict_funct + + + + + + + +if __name__ == "__main__": + #app.run(host="127.0.0.1", port=8080, debug=True) + serve(app, host='0.0.0.0', port=8080) + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/faceitstalker_docker/app/templates/steamids.html b/faceitstalker_docker/app/templates/steamids.html index 756be2c..0df7e69 100644 --- a/faceitstalker_docker/app/templates/steamids.html +++ b/faceitstalker_docker/app/templates/steamids.html @@ -4,30 +4,30 @@
{%for steamid in data_dict%}
- Card image cap + Card image cap
-

{{steam_dict[steamid]['steam_name']}}

+

{{data_dict[steamid]['steam_name']}}

- {% if faceit_dict[steamid]['faceit_acc'] == 'true' %} - {% if faceit_dict[steamid]['faceit_level'] == 1 %} + {% if data_dict[steamid]['faceit_acc'] == 'true' %} + {% if data_dict[steamid]['faceit_level'] == 1 %} FaceIT Level 1 - {% elif faceit_dict[steamid]['faceit_level'] == 2 %} + {% elif data_dict[steamid]['faceit_level'] == 2 %} FaceIT Level 2 - {% elif faceit_dict[steamid]['faceit_level'] == 3 %} + {% elif data_dict[steamid]['faceit_level'] == 3 %} FaceIT Level 3 - {% elif faceit_dict[steamid]['faceit_level'] == 4 %} + {% elif data_dict[steamid]['faceit_level'] == 4 %} FaceIT Level 4 - {% elif faceit_dict[steamid]['faceit_level'] == 5 %} + {% elif data_dict[steamid]['faceit_level'] == 5 %} FaceIT Level 5 - {% elif faceit_dict[steamid]['faceit_level'] == 6 %} + {% elif data_dict[steamid]['faceit_level'] == 6 %} FaceIT Level 6 - {% elif faceit_dict[steamid]['faceit_level'] == 7 %} + {% elif data_dict[steamid]['faceit_level'] == 7 %} FaceIT Level 7 - {% elif faceit_dict[steamid]['faceit_level'] == 8 %} + {% elif data_dict[steamid]['faceit_level'] == 8 %} FaceIT Level 8 - {% elif faceit_dict[steamid]['faceit_level'] == 9 %} + {% elif data_dict[steamid]['faceit_level'] == 9 %} FaceIT Level 9 - {% elif faceit_dict[steamid]['faceit_level'] == 10 %} + {% elif data_dict[steamid]['faceit_level'] == 10 %} FaceIT Level 10 {% endif %} {% else %} @@ -37,14 +37,14 @@

{% if data_dict[steamid]['faceit_acc'] == 'true' %}
-

FaceIT ELO: {{faceit_dict[steamid]['faceit_elo']}}

+

FaceIT ELO: {{data_dict[steamid]['faceit_elo']}}

SteamID: {{data_dict[steamid]['steamid64']}}

{% endif %}
- Steam Profile + Steam Profile {% if data_dict[steamid]['faceit_acc'] == 'true' %} - FaceIT Link + FaceIT Link {% endif %}
diff --git a/wip.py b/wip.py index 5209314..1b33847 100644 --- a/wip.py +++ b/wip.py @@ -13,7 +13,7 @@ import multiprocessing #####Define Global Vars -data_dict = {} + #Input of Status screen. #Kinda disgusting with the fixed lines and really easy to break. @@ -34,30 +34,42 @@ def regex_for_steamids(input_string): steamids_from_status = re.findall(r"\S*STEAM_+[0-9]+:[0-9]+:[0-9]+", input_string) return steamids_from_status -def steamid_to_64bit(steamid): +def get_evererything(steamid): + data_dict_funct = {} + data_dict_funct[steamid] = {} + + #Get Steamid64 steam64id = 76561197960265728 # Who tf knows how this works # IT JUST FUCKING DOES! id_split = steamid.split(":") steam64id += int(id_split[2]) * 2 # JUST MULTIPLY BY 2... Cause THAT MAKES PERFEKT SENSE RIGHT?! if id_split[1] == "1": steam64id += 1 - return steam64id + data_dict_funct[steamid]['steamid64'] = steam64id - -def loop_faceit_data(steamid64): - url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(steamid64) + #Get Faceit Data + url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(data_dict_funct[steamid]['steamid64']) headers = { 'accept': 'application/json', 'Authorization' : 'Bearer ab46a7ab-6ab8-4c00-a8ff-41c0ff71d562' } faceit_response = requests.get(url, headers=headers) faceit_response_json = faceit_response.json() - return faceit_response_json + try: + data_dict_funct[steamid]['faceit_elo'] = faceit_response_json['games']['csgo']['faceit_elo'] + data_dict_funct[steamid]['faceit_level'] = faceit_response_json['games']['csgo']['skill_level'] + data_dict_funct[steamid]['faceit_name'] = faceit_response_json['nickname'] + data_dict_funct[steamid]['faceit_acc'] = 'true' + except: + data_dict_funct[steamid]['faceit_acc'] = 'false' -def loop_steam_data(steamid64): - steamurl ="https://steamcommunity.com/profiles/" + str(steamid64) +"/?xml=1" + #Get Steam Data + steamurl ="https://steamcommunity.com/profiles/" + str(data_dict_funct[steamid]['steamid64']) +"/?xml=1" steam_req = requests.get(steamurl) steamdict_funct = xmltodict.parse(steam_req.content) - return steamdict_funct + data_dict_funct[steamid]['steam_name'] = steamdict_funct['profile']['steamID'] + data_dict_funct[steamid]['steam_pic'] = steamdict_funct['profile']['avatarFull'] + return data_dict_funct + status_screen = get_user_input() steamids = regex_for_steamids(status_screen) @@ -66,11 +78,21 @@ steamids = regex_for_steamids(status_screen) num_cores = multiprocessing.cpu_count() steamid64_dict = {} -steamid64_dict = Parallel(n_jobs=num_cores, verbose=50)(delayed(steamid_to_64bit)(steamid)for steamid in steamids) +faceit_dict = {} +data_dict = {} +#steamid64_dict = Parallel(n_jobs=num_cores, verbose=50)(delayed(steamid_to_64bit)(steamid)for steamid in steamids) +data_dict_list = Parallel(n_jobs=num_cores, verbose=50)(delayed(get_evererything)(steamid)for steamid in steamids) + +for dict in data_dict_list: + for k, v in dict.items(): + data_dict[k] = v -print(steamid64_dict) +for steamid in steamids: + print(data_dict[steamid]) + print("####################") +