Made everything parallel
This commit is contained in:
herz
parent
3e834f9de9
commit
105f565540
|
|
@ -5,8 +5,8 @@ import re
|
|||
import json
|
||||
import xml
|
||||
import xmltodict
|
||||
import asyncio
|
||||
|
||||
from joblib import Parallel, delayed
|
||||
import multiprocessing
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
|
@ -14,7 +14,7 @@ app = Flask(__name__)
|
|||
|
||||
|
||||
@app.route("/", methods=["POST", "GET"])
|
||||
async def index():
|
||||
def index():
|
||||
data_dict = {}
|
||||
|
||||
if request.method == "POST":
|
||||
|
|
@ -22,33 +22,19 @@ async def index():
|
|||
steamids = regex_for_steamids(status_screen)
|
||||
|
||||
|
||||
num_cores = multiprocessing.cpu_count()
|
||||
|
||||
for steamid in steamids:
|
||||
data_dict[steamid] = {}
|
||||
data_dict[steamid]['steamid64'] = steamid_to_64bit(steamid)
|
||||
data_dict = {}
|
||||
#steamid64_dict = Parallel(n_jobs=num_cores, verbose=50)(delayed(steamid_to_64bit)(steamid)for steamid in steamids)
|
||||
data_dict_list = Parallel(n_jobs=num_cores, verbose=50)(delayed(get_evererything)(steamid)for steamid in steamids)
|
||||
|
||||
task_get_faceit_data = asyncio.create_task(loop_faceit_data(data_dict))
|
||||
task_get_steam_data = asyncio.create_task(loop_steam_data(data_dict))
|
||||
for dict in data_dict_list:
|
||||
for k, v in dict.items():
|
||||
data_dict[k] = v
|
||||
|
||||
|
||||
faceit_dict = await task_get_faceit_data
|
||||
steam_dict = await task_get_steam_data
|
||||
|
||||
for steamid in faceit_dict:
|
||||
if faceit_dict[steamid]['faceit_acc'] == 'true' :
|
||||
data_dict[steamid]['faceit_elo'] = faceit_dict[steamid]['faceit_elo']
|
||||
data_dict[steamid]['faceit_level'] = faceit_dict[steamid]['faceit_level']
|
||||
data_dict[steamid]['faceit_name'] = faceit_dict[steamid]['faceit_name']
|
||||
data_dict[steamid]['faceit_acc'] = faceit_dict[steamid]['faceit_acc']
|
||||
else:
|
||||
data_dict[steamid]['faceit_acc'] = faceit_dict[steamid]['faceit_acc']
|
||||
|
||||
for steamid in steam_dict:
|
||||
data_dict[steamid]['steam_name'] = steam_dict[steamid]['steam_name']
|
||||
data_dict[steamid]['steam_pic'] = steam_dict[steamid]['steam_pic']
|
||||
|
||||
|
||||
return render_template('steamids.html', data_dict = data_dict , steam_dict = steam_dict, faceit_dict = faceit_dict)
|
||||
return render_template('steamids.html', data_dict = data_dict)
|
||||
else:
|
||||
return render_template('index.html')
|
||||
|
||||
|
|
@ -63,53 +49,43 @@ def regex_for_steamids(input_string):
|
|||
steamids_from_status = re.findall(r"\S*STEAM_+[0-9]+:[0-9]+:[0-9]+", input_string)
|
||||
return steamids_from_status
|
||||
|
||||
def steamid_to_64bit(steamid):
|
||||
|
||||
|
||||
def get_evererything(steamid):
|
||||
data_dict_funct = {}
|
||||
data_dict_funct[steamid] = {}
|
||||
|
||||
#Get Steamid64
|
||||
steam64id = 76561197960265728 # Who tf knows how this works
|
||||
# IT JUST FUCKING DOES!
|
||||
id_split = steamid.split(":")
|
||||
steam64id += int(id_split[2]) * 2 # JUST MULTIPLY BY 2... Cause THAT MAKES PERFEKT SENSE RIGHT?!
|
||||
if id_split[1] == "1":
|
||||
steam64id += 1
|
||||
return steam64id
|
||||
data_dict_funct[steamid]['steamid64'] = steam64id
|
||||
|
||||
|
||||
async def loop_faceit_data(data_dict):
|
||||
faceit_dict = {}
|
||||
req =requests.Session()
|
||||
for steamid in data_dict:
|
||||
faceit_dict[steamid] = {}
|
||||
url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(data_dict[steamid]['steamid64'])
|
||||
#Get Faceit Data
|
||||
url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(data_dict_funct[steamid]['steamid64'])
|
||||
headers = { 'accept': 'application/json', 'Authorization' : 'Bearer ab46a7ab-6ab8-4c00-a8ff-41c0ff71d562' }
|
||||
faceit_response = req.get(url, headers=headers)
|
||||
faceit_response = requests.get(url, headers=headers)
|
||||
faceit_response_json = faceit_response.json()
|
||||
|
||||
|
||||
try:
|
||||
faceit_dict[steamid]['faceit_elo'] = faceit_response_json['games']['csgo']['faceit_elo']
|
||||
faceit_dict[steamid]['faceit_level'] = faceit_response_json['games']['csgo']['skill_level']
|
||||
faceit_dict[steamid]['faceit_name'] = faceit_response_json['nickname']
|
||||
faceit_dict[steamid]['faceit_acc'] = 'true'
|
||||
data_dict_funct[steamid]['faceit_elo'] = faceit_response_json['games']['csgo']['faceit_elo']
|
||||
data_dict_funct[steamid]['faceit_level'] = faceit_response_json['games']['csgo']['skill_level']
|
||||
data_dict_funct[steamid]['faceit_name'] = faceit_response_json['nickname']
|
||||
data_dict_funct[steamid]['faceit_acc'] = 'true'
|
||||
except:
|
||||
faceit_dict[steamid]['faceit_acc'] = 'false'
|
||||
data_dict_funct[steamid]['faceit_acc'] = 'false'
|
||||
|
||||
return faceit_dict
|
||||
|
||||
async def loop_steam_data(data_dict):
|
||||
steam_dict = {}
|
||||
req =requests.Session()
|
||||
for steamid in data_dict:
|
||||
steam_dict[steamid] = {}
|
||||
steamurl ="https://steamcommunity.com/profiles/" + str(data_dict[steamid]['steamid64']) +"/?xml=1"
|
||||
steam_req = req.get(steamurl)
|
||||
#Get Steam Data
|
||||
steamurl ="https://steamcommunity.com/profiles/" + str(data_dict_funct[steamid]['steamid64']) +"/?xml=1"
|
||||
steam_req = requests.get(steamurl)
|
||||
steamdict_funct = xmltodict.parse(steam_req.content)
|
||||
|
||||
steam_dict[steamid]['steam_name'] = steamdict_funct['profile']['steamID']
|
||||
steam_dict[steamid]['steam_pic'] = steamdict_funct['profile']['avatarFull']
|
||||
|
||||
return steam_dict
|
||||
|
||||
data_dict_funct[steamid]['steam_name'] = steamdict_funct['profile']['steamID']
|
||||
data_dict_funct[steamid]['steam_pic'] = steamdict_funct['profile']['avatarFull']
|
||||
|
||||
|
||||
return data_dict_funct
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,122 @@
|
|||
from flask import Flask, render_template, request, redirect, url_for
|
||||
from waitress import serve
|
||||
import requests
|
||||
import re
|
||||
import json
|
||||
import xml
|
||||
import xmltodict
|
||||
from joblib import Parallel, delayed
|
||||
import multiprocessing
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
|
||||
|
||||
@app.route("/", methods=["POST", "GET"])
|
||||
def index():
|
||||
data_dict = {}
|
||||
|
||||
if request.method == "POST":
|
||||
status_screen = request.form["status_screen_input"]
|
||||
steamids = regex_for_steamids(status_screen)
|
||||
|
||||
|
||||
num_cores = multiprocessing.cpu_count()
|
||||
|
||||
data_dict = {}
|
||||
#steamid64_dict = Parallel(n_jobs=num_cores, verbose=50)(delayed(steamid_to_64bit)(steamid)for steamid in steamids)
|
||||
data_dict_list = Parallel(n_jobs=num_cores, verbose=50)(delayed(get_evererything)(steamid)for steamid in steamids)
|
||||
|
||||
for dict in data_dict_list:
|
||||
for k, v in dict.items():
|
||||
data_dict[k] = v
|
||||
|
||||
|
||||
|
||||
return render_template('steamids.html', data_dict = data_dict)
|
||||
else:
|
||||
return render_template('index.html')
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def regex_for_steamids(input_string):
|
||||
steamids_from_status = re.findall(r"\S*STEAM_+[0-9]+:[0-9]+:[0-9]+", input_string)
|
||||
return steamids_from_status
|
||||
|
||||
|
||||
|
||||
def get_evererything(steamid):
|
||||
data_dict_funct = {}
|
||||
data_dict_funct[steamid] = {}
|
||||
|
||||
#Get Steamid64
|
||||
steam64id = 76561197960265728 # Who tf knows how this works
|
||||
# IT JUST FUCKING DOES!
|
||||
id_split = steamid.split(":")
|
||||
steam64id += int(id_split[2]) * 2 # JUST MULTIPLY BY 2... Cause THAT MAKES PERFEKT SENSE RIGHT?!
|
||||
if id_split[1] == "1":
|
||||
steam64id += 1
|
||||
data_dict_funct[steamid]['steamid64'] = steam64id
|
||||
|
||||
#Get Faceit Data
|
||||
url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(data_dict_funct[steamid]['steamid64'])
|
||||
headers = { 'accept': 'application/json', 'Authorization' : 'Bearer ab46a7ab-6ab8-4c00-a8ff-41c0ff71d562' }
|
||||
faceit_response = requests.get(url, headers=headers)
|
||||
faceit_response_json = faceit_response.json()
|
||||
try:
|
||||
data_dict_funct[steamid]['faceit_elo'] = faceit_response_json['games']['csgo']['faceit_elo']
|
||||
data_dict_funct[steamid]['faceit_level'] = faceit_response_json['games']['csgo']['skill_level']
|
||||
data_dict_funct[steamid]['faceit_name'] = faceit_response_json['nickname']
|
||||
data_dict_funct[steamid]['faceit_acc'] = 'true'
|
||||
except:
|
||||
data_dict_funct[steamid]['faceit_acc'] = 'false'
|
||||
|
||||
#Get Steam Data
|
||||
steamurl ="https://steamcommunity.com/profiles/" + str(data_dict_funct[steamid]['steamid64']) +"/?xml=1"
|
||||
steam_req = requests.get(steamurl)
|
||||
steamdict_funct = xmltodict.parse(steam_req.content)
|
||||
data_dict_funct[steamid]['steam_name'] = steamdict_funct['profile']['steamID']
|
||||
data_dict_funct[steamid]['steam_pic'] = steamdict_funct['profile']['avatarFull']
|
||||
|
||||
|
||||
return data_dict_funct
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
#app.run(host="127.0.0.1", port=8080, debug=True)
|
||||
serve(app, host='0.0.0.0', port=8080)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -4,30 +4,30 @@
|
|||
<div class="input-group">
|
||||
{%for steamid in data_dict%}
|
||||
<div class="card" >
|
||||
<img class="Steam-PB" src="{{steam_dict[steamid]['steam_pic']}}" alt="Card image cap">
|
||||
<img class="Steam-PB" src="{{data_dict[steamid]['steam_pic']}}" alt="Card image cap">
|
||||
<div class="card-body">
|
||||
<h3 class="Steam-Name">{{steam_dict[steamid]['steam_name']}}</h3>
|
||||
<h3 class="Steam-Name">{{data_dict[steamid]['steam_name']}}</h3>
|
||||
<p class="FaceIT-IMG">
|
||||
{% if faceit_dict[steamid]['faceit_acc'] == 'true' %}
|
||||
{% if faceit_dict[steamid]['faceit_level'] == 1 %}
|
||||
{% if data_dict[steamid]['faceit_acc'] == 'true' %}
|
||||
{% if data_dict[steamid]['faceit_level'] == 1 %}
|
||||
<img src="/static/faceit1.svg" alt="FaceIT Level 1" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 2 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 2 %}
|
||||
<img src="/static/faceit2.svg" alt="FaceIT Level 2" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 3 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 3 %}
|
||||
<img src="/static/faceit3.svg" alt="FaceIT Level 3" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 4 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 4 %}
|
||||
<img src="/static/faceit4.svg" alt="FaceIT Level 4" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 5 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 5 %}
|
||||
<img src="/static/faceit5.svg" alt="FaceIT Level 5" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 6 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 6 %}
|
||||
<img src="/static/faceit6.svg" alt="FaceIT Level 6" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 7 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 7 %}
|
||||
<img src="/static/faceit7.svg" alt="FaceIT Level 7" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 8 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 8 %}
|
||||
<img src="/static/faceit8.svg" alt="FaceIT Level 8" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 9 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 9 %}
|
||||
<img src="/static/faceit9.svg" alt="FaceIT Level 9" width="100" height="100">
|
||||
{% elif faceit_dict[steamid]['faceit_level'] == 10 %}
|
||||
{% elif data_dict[steamid]['faceit_level'] == 10 %}
|
||||
<img src="/static/faceit10.svg" alt="FaceIT Level 10" width="100" height="100">
|
||||
{% endif %}
|
||||
{% else %}
|
||||
|
|
@ -37,14 +37,14 @@
|
|||
</div>
|
||||
{% if data_dict[steamid]['faceit_acc'] == 'true' %}
|
||||
<div class="Faceit-elo-and-steamid64">
|
||||
<p class="FaceIT-elo">FaceIT ELO: {{faceit_dict[steamid]['faceit_elo']}}</p>
|
||||
<p class="FaceIT-elo">FaceIT ELO: {{data_dict[steamid]['faceit_elo']}}</p>
|
||||
<p class="SteamID64">SteamID: {{data_dict[steamid]['steamid64']}}</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="profile-links">
|
||||
<a href="https://steamcommunity.com/profiles/{{steam_dict[steamid]['steamid64']}}" class="card-link">Steam Profile</a>
|
||||
<a href="https://steamcommunity.com/profiles/{{data_dict[steamid]['steamid64']}}" class="card-link">Steam Profile</a>
|
||||
{% if data_dict[steamid]['faceit_acc'] == 'true' %}
|
||||
<a href="https://www.faceit.com/en/players/{{faceit_dict[steamid]['faceit_name']}}" class="card-link">FaceIT Link</a>
|
||||
<a href="https://www.faceit.com/en/players/{{data_dict[steamid]['faceit_name']}}" class="card-link">FaceIT Link</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
46
wip.py
46
wip.py
|
|
@ -13,7 +13,7 @@ import multiprocessing
|
|||
|
||||
#####Define Global Vars
|
||||
|
||||
data_dict = {}
|
||||
|
||||
|
||||
#Input of Status screen.
|
||||
#Kinda disgusting with the fixed lines and really easy to break.
|
||||
|
|
@ -34,30 +34,42 @@ def regex_for_steamids(input_string):
|
|||
steamids_from_status = re.findall(r"\S*STEAM_+[0-9]+:[0-9]+:[0-9]+", input_string)
|
||||
return steamids_from_status
|
||||
|
||||
def steamid_to_64bit(steamid):
|
||||
def get_evererything(steamid):
|
||||
data_dict_funct = {}
|
||||
data_dict_funct[steamid] = {}
|
||||
|
||||
#Get Steamid64
|
||||
steam64id = 76561197960265728 # Who tf knows how this works
|
||||
# IT JUST FUCKING DOES!
|
||||
id_split = steamid.split(":")
|
||||
steam64id += int(id_split[2]) * 2 # JUST MULTIPLY BY 2... Cause THAT MAKES PERFEKT SENSE RIGHT?!
|
||||
if id_split[1] == "1":
|
||||
steam64id += 1
|
||||
return steam64id
|
||||
data_dict_funct[steamid]['steamid64'] = steam64id
|
||||
|
||||
|
||||
def loop_faceit_data(steamid64):
|
||||
url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(steamid64)
|
||||
#Get Faceit Data
|
||||
url = "https://open.faceit.com/data/v4/players?game=csgo&game_player_id=" + str(data_dict_funct[steamid]['steamid64'])
|
||||
headers = { 'accept': 'application/json', 'Authorization' : 'Bearer ab46a7ab-6ab8-4c00-a8ff-41c0ff71d562' }
|
||||
faceit_response = requests.get(url, headers=headers)
|
||||
faceit_response_json = faceit_response.json()
|
||||
return faceit_response_json
|
||||
try:
|
||||
data_dict_funct[steamid]['faceit_elo'] = faceit_response_json['games']['csgo']['faceit_elo']
|
||||
data_dict_funct[steamid]['faceit_level'] = faceit_response_json['games']['csgo']['skill_level']
|
||||
data_dict_funct[steamid]['faceit_name'] = faceit_response_json['nickname']
|
||||
data_dict_funct[steamid]['faceit_acc'] = 'true'
|
||||
except:
|
||||
data_dict_funct[steamid]['faceit_acc'] = 'false'
|
||||
|
||||
def loop_steam_data(steamid64):
|
||||
steamurl ="https://steamcommunity.com/profiles/" + str(steamid64) +"/?xml=1"
|
||||
#Get Steam Data
|
||||
steamurl ="https://steamcommunity.com/profiles/" + str(data_dict_funct[steamid]['steamid64']) +"/?xml=1"
|
||||
steam_req = requests.get(steamurl)
|
||||
steamdict_funct = xmltodict.parse(steam_req.content)
|
||||
return steamdict_funct
|
||||
data_dict_funct[steamid]['steam_name'] = steamdict_funct['profile']['steamID']
|
||||
data_dict_funct[steamid]['steam_pic'] = steamdict_funct['profile']['avatarFull']
|
||||
|
||||
|
||||
return data_dict_funct
|
||||
|
||||
|
||||
status_screen = get_user_input()
|
||||
steamids = regex_for_steamids(status_screen)
|
||||
|
|
@ -66,11 +78,21 @@ steamids = regex_for_steamids(status_screen)
|
|||
num_cores = multiprocessing.cpu_count()
|
||||
|
||||
steamid64_dict = {}
|
||||
steamid64_dict = Parallel(n_jobs=num_cores, verbose=50)(delayed(steamid_to_64bit)(steamid)for steamid in steamids)
|
||||
faceit_dict = {}
|
||||
data_dict = {}
|
||||
#steamid64_dict = Parallel(n_jobs=num_cores, verbose=50)(delayed(steamid_to_64bit)(steamid)for steamid in steamids)
|
||||
data_dict_list = Parallel(n_jobs=num_cores, verbose=50)(delayed(get_evererything)(steamid)for steamid in steamids)
|
||||
|
||||
for dict in data_dict_list:
|
||||
for k, v in dict.items():
|
||||
data_dict[k] = v
|
||||
|
||||
|
||||
|
||||
print(steamid64_dict)
|
||||
for steamid in steamids:
|
||||
print(data_dict[steamid])
|
||||
print("####################")
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue