data/stats_importer.py

@@ -1,6 +1,9 @@
 import os
 import csv
+import time
 import shutil
+
+from math import *
 from data.db_connect import Database
 from data.build_weather import get_weather, get_sun_and_moon_phase
 
@@ -42,7 +45,21 @@ class Importer:
             reader = csv.DictReader(bb_data_file)
             bb_dict = list(reader)
 
+        count = 0
         for game in bb_dict:
+            # Delay to not overwhelm the free api
+            count += 1
+            print(f"Current line {count}")
+            if count % 600 == 0:
+                print("Sleeping for 1 min")
+                time.sleep(60)
+            if count % 5000 == 0:
+                print("Sleeping for 1 hour")
+                time.sleep(60*60)
+            if count % 10000 == 0:
+                print("Sleeping for 1 day")
+                time.sleep(60*60*24)
+                
             if not self.populate_database_with_stats(game):
                 return False            
         
@@ -56,6 +73,11 @@ class Importer:
             print(f"{parkid} is None")
             return True
         
+        check_game_added_query = "SELECT id FROM games WHERE game_date = ? AND game_number = ? AND park_id = ?"
+        check_game_added_data = [game_stats["date"], game_stats["num-of-game"], game_stats['park-id']]
+        if self.database.select(check_game_added_query, check_game_added_data) is not None:
+            return True
+
         insert_game = """
         INSERT INTO games 
             (   
@@ -160,13 +182,15 @@ class Importer:
 
         if "error" in historic_weather:
             print(f"Error: {historic_weather['error']}: Details: {historic_weather['details']}")
-            return False
+            if "No weather data available" in historic_weather['details']:
-
+                historic_weather = None
-        if "hourly" not in historic_weather:
+            else:
+                return False
+        elif "hourly" not in historic_weather:
             print(f"Failed to get weather: Full JSON: {historic_weather}")
-            return False
+            historic_weather = None
-
+        else:
-        historic_weather = historic_weather["hourly"]
+            historic_weather = historic_weather["hourly"]
 
         game_data = [
             game_stats["date"], game_stats["num-of-game"], game_stats["day-of-week"],
@@ -234,17 +258,18 @@ class Importer:
         self.database.insert(insert_team_game, visiting_team_data)
         self.database.insert(insert_team_game, home_team_data)
 
-        (sunrise_time, sunset_time, moonphase) = get_sun_and_moon_phase(park_data[0], park_data[1], game_stats["date"])
+        if historic_weather is not None:
+            (sunrise_time, sunset_time, moonphase) = get_sun_and_moon_phase(park_data[0], park_data[1], game_stats["date"])
 
-        weather_data = [
+            weather_data = [
-            game_id, historic_weather["temperature_2m"][hour], historic_weather["relative_humidity_2m"][hour],
+                game_id, historic_weather["temperature_2m"][hour], historic_weather["relative_humidity_2m"][hour],
-            historic_weather["dew_point_2m"][hour], historic_weather["apparent_temperature"][hour], historic_weather["pressure_msl"][hour],
+                historic_weather["dew_point_2m"][hour], historic_weather["apparent_temperature"][hour], historic_weather["pressure_msl"][hour],
-            historic_weather["precipitation"][hour], historic_weather["rain"][hour], historic_weather["snowfall"][hour], 
+                historic_weather["precipitation"][hour], historic_weather["rain"][hour], historic_weather["snowfall"][hour], 
-            historic_weather["cloud_cover"][hour], historic_weather["wind_speed_10m"][hour], historic_weather["wind_direction_10m"][hour],
+                historic_weather["cloud_cover"][hour], historic_weather["wind_speed_10m"][hour], historic_weather["wind_direction_10m"][hour],
-            historic_weather["wind_gusts_10m"][hour], sunrise_time, sunset_time,
+                historic_weather["wind_gusts_10m"][hour], sunrise_time, sunset_time,
-            moonphase,
+                moonphase,
-        ]
+            ]
 
-        self.database.insert(insert_into_weather, weather_data)
+            self.database.insert(insert_into_weather, weather_data)
 
         return True