paul/baseball-nn
1
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Throttling connections for free API, Skipping already imported games

Browse Source
This commit is contained in:
paul
2025-08-26 19:27:59 +00:00
parent f59ae02734
commit ecaad438a5
2 changed files with 566 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

951
Pipfile.lock generated
View File

File diff suppressed because it is too large Load Diff

59
data/stats_importer.py
View File

@@ -1,6 +1,9 @@
import os
import csv
import time
import shutil
from math import *
from data.db_connect import Database
from data.build_weather import get_weather, get_sun_and_moon_phase
@@ -42,9 +45,23 @@ class Importer:
reader = csv.DictReader(bb_data_file)
bb_dict = list(reader)
count = 0
for game in bb_dict:
# Delay to not overwhelm the free api
count += 1
print(f"Current line {count}")
if count % 600 == 0:
print("Sleeping for 1 min")
time.sleep(60)
if count % 5000 == 0:
print("Sleeping for 1 hour")
time.sleep(60*60)
if count % 10000 == 0:
print("Sleeping for 1 day")
time.sleep(60*60*24)
if not self.populate_database_with_stats(game):
return False
return False
return True
@@ -56,6 +73,11 @@ class Importer:
print(f"{parkid} is None")
return True
check_game_added_query = "SELECT id FROM games WHERE game_date = ? AND game_number = ? AND park_id = ?"
check_game_added_data = [game_stats["date"], game_stats["num-of-game"], game_stats['park-id']]
if self.database.select(check_game_added_query, check_game_added_data) is not None:
return True
insert_game = """
INSERT INTO games
(
@@ -160,13 +182,15 @@ class Importer:
if "error" in historic_weather:
print(f"Error: {historic_weather['error']}: Details: {historic_weather['details']}")
return False
if "hourly" not in historic_weather:
if "No weather data available" in historic_weather['details']:
historic_weather = None
else:
return False
elif "hourly" not in historic_weather:
print(f"Failed to get weather: Full JSON: {historic_weather}")
return False
historic_weather = historic_weather["hourly"]
historic_weather = None
else:
historic_weather = historic_weather["hourly"]
game_data = [
game_stats["date"], game_stats["num-of-game"], game_stats["day-of-week"],
@@ -234,17 +258,18 @@ class Importer:
self.database.insert(insert_team_game, visiting_team_data)
self.database.insert(insert_team_game, home_team_data)
(sunrise_time, sunset_time, moonphase) = get_sun_and_moon_phase(park_data[0], park_data[1], game_stats["date"])
if historic_weather is not None:
(sunrise_time, sunset_time, moonphase) = get_sun_and_moon_phase(park_data[0], park_data[1], game_stats["date"])
weather_data = [
game_id, historic_weather["temperature_2m"][hour], historic_weather["relative_humidity_2m"][hour],
historic_weather["dew_point_2m"][hour], historic_weather["apparent_temperature"][hour], historic_weather["pressure_msl"][hour],
historic_weather["precipitation"][hour], historic_weather["rain"][hour], historic_weather["snowfall"][hour],
historic_weather["cloud_cover"][hour], historic_weather["wind_speed_10m"][hour], historic_weather["wind_direction_10m"][hour],
historic_weather["wind_gusts_10m"][hour], sunrise_time, sunset_time,
moonphase,
]
weather_data = [
game_id, historic_weather["temperature_2m"][hour], historic_weather["relative_humidity_2m"][hour],
historic_weather["dew_point_2m"][hour], historic_weather["apparent_temperature"][hour], historic_weather["pressure_msl"][hour],
historic_weather["precipitation"][hour], historic_weather["rain"][hour], historic_weather["snowfall"][hour],
historic_weather["cloud_cover"][hour], historic_weather["wind_speed_10m"][hour], historic_weather["wind_direction_10m"][hour],
historic_weather["wind_gusts_10m"][hour], sunrise_time, sunset_time,
moonphase,
]
self.database.insert(insert_into_weather, weather_data)
self.database.insert(insert_into_weather, weather_data)
return True