Files
solon/scrape_data.py
T
2023-04-12 23:25:00 +02:00

511 lines
12 KiB
Python

import requests
from bs4 import BeautifulSoup
import time
session = requests.Session()
REMOTE_SERVER = "one.one.one.one"
not_login = 0
F_not_anymore = 0
def fetch_locally(what: str) -> list:
global not_login, F_not_anymore
if not_login == 1 and F_not_anymore == 0:
F_not_anymore = 1
not_login = 0
print("Could not sign in (you may have entered invalid login credentials)")
print("Fetching locally\n")
if what == "schedule":
schedule = schedule_from_file()
return schedule
elif what == "grades":
grades = grades_from_file()
return grades
def login(user: str, pwd: str) -> bool:
if is_connected(REMOTE_SERVER) == True:
url: str = "https://aplikace.skolaonline.cz/SOL/Prihlaseni.aspx"
data = {
"__EVENTTARGET": "dnn$ctr994$SOLLogin$btnODeslat",
"__EVENTARGUMENT": "",
"__VIEWSTATE": "",
"__VIEWSTATEGENERATOR": "",
"__VIEWSTATEENCRYPTED": "",
"__PREVIOUSPAGE": "",
"__EVENTVALIDATION": "",
"dnn$dnnSearch$txtSearch": "",
"JmenoUzivatele": user, # username
"HesloUzivatele": pwd, # password
"ScrollTop": "",
"__dnnVariable": "",
"__RequestVerificationToken": ""
}
post_login = session.post(url, data=data)
if post_login.status_code != 200:
#print("Could not sign in")
return False
else:
#print("Sign in successful")
return True
else:
#print("Could not sign in")
#print("Fetching data from local files")
return False
def is_connected(hostname: str) -> bool:
import socket
try:
host = socket.gethostbyname(hostname)
s = socket.create_connection((host, 80), 2)
s.close()
return True
except Exception:
pass
return False
def get_html_from(params=None) -> str:
base = "https://aplikace.skolaonline.cz"
if params is not None:
base += params
response = session.get(base)
if response.status_code == 200:
#print("Download successful")
return response
else:
print("Download unsuccessful")
return ""
def sort_calendar(cal: list[dict]) -> list[dict]:
sorted_cal: list[dict] = [{}, {}, {}, {}, {}]
for i in range(len(cal)):
if cal[i]["DAY"] == "Po":
sorted_cal[0] = cal[i]
if cal[i]["DAY"] == "Út":
sorted_cal[1] = cal[i]
if cal[i]["DAY"] == "St":
sorted_cal[2] = cal[i]
if cal[i]["DAY"] == "Čt":
sorted_cal[3] = cal[i]
if cal[i]["DAY"] == "Pá":
sorted_cal[4] = cal[i]
return sorted_cal
def get_schedule(url: str):
try:
html = get_html_from(url)
soup = BeautifulSoup(html.text, "lxml")
table = soup.find("table", id="CCADynamicCalendarTable")
rows_odd = table.find_all("tr", class_="RowOdd")
rows_even = table.find_all("tr", class_="RowEven")
rows = rows_odd + rows_even
calendar = []
for i in range(len(rows)):
if rows[i].find("td", class_="KuvSuplovanaHodina") is not None and rows[i].find("td", class_="KuvSuplujiciHodina") is None:
continue
subjects = []
wheres = []
cells = rows[i].find_all("td", attrs={"class": "DctCell"})
for j in range(len(cells)):
if cells[j].text == "":
new_span = soup.new_tag("span", attrs={'class': 'KuvBunkaRozvrhNadpis'})
new_span.string = "Volno"
cells[j].append(new_span)
if rows[i].find("td", class_="KuvHeaderNadpis") is not None:
day = rows[i].find("td", class_="KuvHeaderNadpis").text
date = rows[i].find("td", class_="KuvHeaderText").text
subjects_ = rows[i].find_all("span", class_="KuvBunkaRozvrhNadpis")
temp = []
for j in range(len(subjects_)):
temp.append(subjects_[j].text)
subjects = temp
F_con = False
try:
while subjects[-1] == "Volno":
del subjects[-1]
if subjects == []:
F_con = True
except IndexError:
pass
wheres_ = rows[i].find_all("span", class_="KuvBunkaRozvrhText")
temp = []
for j in range(len(wheres_)):
if len(subjects[j]) < 10:
temp.append(wheres_[j].text[4:])
else:
temp.append(" - ")
wheres = temp
for j in range(len(cells)):
if cells[j].get("colspan") is not None:
if int(cells[j].get("colspan")) == 2:
subjects.insert(j, subjects[j])
wheres.insert(j, wheres[j])
for j in range(len(subjects)):
if subjects[j] == "Volno":
wheres.insert(j, " - ")
if F_con == False:
calendar_row = {
"DAY": day,
"DATE": date,
"SUBJECTS": subjects,
"WHERES": wheres
}
calendar.append(calendar_row)
sorted_calendar = sort_calendar(calendar)
return sorted_calendar
except AttributeError:
global not_login
not_login = 1
schedule = fetch_locally("schedule")
return schedule
def schedule_from_file_to_list():
schedule = []
schedule_ = []
try:
with open("schedule.csv") as f:
import csv
reader = csv.reader(f)
for row in reader:
temp = []
for col in row:
temp.append(col.strip())
schedule_.append(temp)
for i in range(0, len(schedule_), 2):
del schedule_[i][0]
temp = []
for i in range(1, len(schedule_), 2):
temp.append(schedule_[i][0])
del schedule_[i][0]
schedule.append(temp)
for i in range(10):
temp = []
for j in range(10):
try:
temp.append(schedule_[j][0])
del schedule_[j][0]
except IndexError:
pass
schedule.append(temp)
except FileNotFoundError:
print("No schedule.csv file found\n")
return schedule
def get_whitespace(key: str, idx=0) -> int:
schedule = schedule_from_file_to_list()
max_len = 0
if key == "DATE":
try:
max_len = len(max(schedule[0], key=len))
except ValueError:
pass
elif key == "WHERES":
try:
max_len = len(max(schedule[idx], key=len))
except ValueError:
pass
return max_len
def show_schedule():
# if there is internet access, load stuff from the web
# if not, load the schedule.csv file
if is_connected(REMOTE_SERVER) == True:
schedule = get_schedule("/SOL/App/Kalendar/KZK001_KalendarTyden.aspx")
schedule_to_file(schedule)
else:
schedule = schedule_from_file()
print("--------")
print(" Rozvrh")
print("--------\n")
if schedule == []:
print("N/A\n")
import os
if os.path.exists("schedule.csv"):
os.remove("schedule.csv")
return
for i in range(len(schedule)):
max_len = get_whitespace("DATE")
whitespace = " " * (max_len - len(schedule[i]["DAY"]))
print(schedule[i]["DAY"] + whitespace, end=" ")
for j in range(len(schedule[i]["SUBJECTS"])):
max_len = get_whitespace("WHERES", j+1)
whitespace = " " * (max_len - len(schedule[i]["SUBJECTS"][j]))
print("| " + schedule[i]["SUBJECTS"][j] + whitespace, end=" ")
print("")
max_len = get_whitespace("DATE")
whitespace = " " * (max_len - len(schedule[i]["DATE"]))
print(schedule[i]["DATE"] + whitespace, end=" ")
for j in range(len(schedule[i]["WHERES"])):
max_len = get_whitespace("WHERES", j+1)
whitespace = " " * (max_len - len(schedule[i]["WHERES"][j]))
print("| " + schedule[i]["WHERES"][j] + whitespace, end=" ")
print("")
print("")
def get_avg(grades: list) -> float:
if len(grades) < 1:
return ""
sum_ = 0
for i in grades:
sum_ += int(i)
avg = round(sum_ / len(grades), 2)
return str(avg)
def get_grades(url: str):
html = get_html_from(url)
soup = BeautifulSoup(html.text, "lxml")
rows = soup.find("table", id="G_ctl00xmainxCCADynamicUWGrid").find_all("tr")
for i in range(3):
r = soup.find("table", id="G_ctl00xmainxCCADynamicUWGrid").find("tr")
r.decompose()
grades = []
for i in range(1, len(rows)):
if rows[i].find("td") is not None:
subject = rows[i].find("td").text
temp_dict = {}
temp_dict["SUBJECT"] = subject
cell = rows[i].find_all("span", class_="VHVysl")
temp = []
for j in range(len(cell)):
temp.append(cell[j].text)
temp_dict["GRADES"] = temp
grades.append(temp_dict)
return grades
def show_grades():
if is_connected(REMOTE_SERVER) == True:
grades = get_grades("/SOL/App/Hodnoceni/KZH001_HodnVypisStud.aspx")
grades_to_file(grades)
else:
grades = grades_from_file()
whitespace = ""
max_len_of_subject = 0
subjects = [grades[j]["SUBJECT"] for j in range(len(grades))]
for i in subjects:
if len(i) > max_len_of_subject:
max_len_of_subject = len(i)
max_len_of_subject += 1
print("--------")
print(" Znamky")
print("--------\n")
if grades == []:
print("N/A\n")
import os
if os.path.exists("grades.csv"):
os.remove("grades.csv")
return
for i in range(len(grades)):
whitespace = " " * (max_len_of_subject - len(grades[i]["SUBJECT"]))
print(grades[i]["SUBJECT"] + whitespace + "|", end=" ")
print("".join([grades[i]["GRADES"][j]+" " for j in range(len(grades[i]["GRADES"]))]), end=" ")
print("|", get_avg(grades[i]["GRADES"]))
print("")
print("")
def schedule_to_file(schedule=None):
import csv
if schedule is None:
try:
schedule = get_schedule("/SOL/App/Kalendar/KZK001_KalendarTyden.aspx")
except Exception:
print("An error occured, you probably can't access the internet right now")
return
with open("schedule.csv", mode="w", newline="") as f:
data = []
for i in range(len(schedule)):
temp = []
try:
temp.append(schedule[i]["DAY"])
for j in range(len(schedule[i]["SUBJECTS"])):
temp.append(schedule[i]["SUBJECTS"][j])
data.append(temp)
temp = []
temp.append(schedule[i]["DATE"])
for j in range(len(schedule[i]["WHERES"])):
temp.append(schedule[i]["WHERES"][j])
data.append(temp)
temp = []
for j in range(len(schedule[i]["WHOS"])):
temp.append(schedule[i]["WHOS"][j])
data.append(temp)
except KeyError:
pass
writer = csv.writer(f)
for row in data:
writer.writerow(row)
def schedule_from_file() -> list:
import csv
schedule = []
try:
with open("schedule.csv", mode="r") as f:
reader = csv.reader(f)
rows = []
for row in reader:
rows.append(row)
for row in range(0, len(rows), 3):
temp = {}
temp["DAY"] = rows[row][0]
try:
temp["DATE"] = rows[row+1][0]
except IndexError:
pass
subjects = []
for i in range(1, len(rows[row])):
subjects.append(rows[row][i])
temp["SUBJECTS"] = subjects
wheres = []
try:
for i in range(1, len(rows[row+1])):
wheres.append(rows[row+1][i])
except IndexError:
pass
temp["WHERES"] = wheres
whos = []
try:
for i in range(len(rows[row+2])):
whos.append(rows[row+2][i])
except IndexError:
pass
temp["WHOS"] = whos
schedule.append(temp)
except FileNotFoundError:
print("No schedule.csv file found\n")
return schedule
def grades_to_file(grades=None): # !!! make sure the format of the grades list is saved correctly !!!
import csv
if grades is None:
try:
grades = get_grades("/SOL/App/Hodnoceni/KZH001_HodnVypisStud.aspx")
except Exception:
print("An error occured, you probably can't access the internet right now")
return
data = []
with open("grades.csv", mode="w", newline="") as f:
for i in range(len(grades)):
temp = {}
temp["SUBJECT"] = grades[i]["SUBJECT"]
temp_grades = []
for j in range(len(grades[i]["GRADES"])):
temp_grades.append(grades[i]["GRADES"][j])
temp["GRADES"] = temp_grades
data.append(temp)
writer = csv.writer(f)
for row in data:
_row = []
_row.append(row["SUBJECT"])
for i in row["GRADES"]:
_row.append(i)
writer.writerow(_row)
def grades_from_file() -> list:
import csv
grades = []
try:
with open("grades.csv", mode="r") as f:
reader = csv.reader(f)
rows = []
for row in reader:
rows.append(row)
for row in range(len(rows)):
temp = {}
temp["SUBJECT"] = rows[row][0]
temp_lst = []
for i in range(1, len(rows[row])):
temp_lst.append(rows[row][i])
temp["GRADES"] = temp_lst
grades.append(temp)
except FileNotFoundError:
print("No grades.csv file found\n")
return grades
if __name__ == "__main__":
user = str(input("Zadejte prihlasovaci jmeno: "))
pwd = str(input("Zadejte heslo: "))
if is_connected(REMOTE_SERVER) == True:
login(user, pwd)
#get_schedule("/SOL/App/Kalendar/KZK001_KalendarTyden.aspx")
show_schedule()
#get_grades("/SOL/App/Hodnoceni/KZH001_HodnVypisStud.aspx")
show_grades()
#schedule_to_file()
#grades_to_file()
#schedule_from_file()