data/stats_importer.py

@@ -1,6 +1,9 @@
 import os
 import csv
+import time
 import shutil
+
+from math import *
 from data.db_connect import Database
 from data.build_weather import get_weather, get_sun_and_moon_phase
 
@@ -42,7 +45,21 @@ class Importer:
             reader = csv.DictReader(bb_data_file)
             bb_dict = list(reader)
 
+        count = 0
         for game in bb_dict:
+            # Delay to not overwhelm the free api
+            count += 1
+            print(f"Current line {count}")
+            if count % 600 == 0:
+                print("Sleeping for 1 min")
+                time.sleep(60)
+            if count % 5000 == 0:
+                print("Sleeping for 1 hour")
+                time.sleep(60*60)
+            if count % 10000 == 0:
+                print("Sleeping for 1 day")
+                time.sleep(60*60*24)
+                
             if not self.populate_database_with_stats(game):
                 return False            
         
@@ -56,6 +73,11 @@ class Importer:
             print(f"{parkid} is None")
             return True
         
+        check_game_added_query = "SELECT id FROM games WHERE game_date = ? AND game_number = ? AND park_id = ?"
+        check_game_added_data = [game_stats["date"], game_stats["num-of-game"], game_stats['park-id']]
+        if self.database.select(check_game_added_query, check_game_added_data) is not None:
+            return True
+
         insert_game = """
         INSERT INTO games 
             (   
@@ -160,12 +182,14 @@ class Importer:
 
         if "error" in historic_weather:
             print(f"Error: {historic_weather['error']}: Details: {historic_weather['details']}")
+            if "No weather data available" in historic_weather['details']:
+                historic_weather = None
+            else:
                 return False
-
-        if "hourly" not in historic_weather:
+        elif "hourly" not in historic_weather:
             print(f"Failed to get weather: Full JSON: {historic_weather}")
-            return False
-
+            historic_weather = None
+        else:
             historic_weather = historic_weather["hourly"]
 
         game_data = [
@@ -234,6 +258,7 @@ class Importer:
         self.database.insert(insert_team_game, visiting_team_data)
         self.database.insert(insert_team_game, home_team_data)
 
+        if historic_weather is not None:
             (sunrise_time, sunset_time, moonphase) = get_sun_and_moon_phase(park_data[0], park_data[1], game_stats["date"])
 
             weather_data = [