Refactor covid_hosp auto columns (#1203)

reworked common.Database to be more reflective/introspective w/ CovidHospSomething (YAML representation accessor), and more useful as an object instead of a class, and pulled appropriate methods into it that were previously in a "utils" file.  greatly simplified other common.Xxx subclasses, and rearranged lots of tests.

Co-authored-by: melange396 <[email protected]>

Author: rzats

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -2,14 +2,14 @@
 
 # standard library
 import unittest
-from unittest.mock import MagicMock
+from unittest.mock import patch
 
 # first party
-from delphi.epidata.acquisition.covid_hosp.common.database import Database
-from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
+from delphi.epidata.acquisition.covid_hosp.facility.database import Database
+from delphi.epidata.acquisition.covid_hosp.common.network import Network
+from delphi.epidata.acquisition.covid_hosp.common.test_utils import CovidHospTestCase, UnitTestUtils
 from delphi.epidata.client.delphi_epidata import Epidata
-from delphi.epidata.acquisition.covid_hosp.facility.update import Update
-import delphi.operations.secrets as secrets
+from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething
 
 # third party
 from freezegun import freeze_time
@@ -19,48 +19,26 @@
 
 NEWLINE="\n"
 
-class AcquisitionTests(unittest.TestCase):
+class AcquisitionTests(CovidHospTestCase):
 
-  def setUp(self):
-    """Perform per-test setup."""
-
-    # configure test data
-    self.test_utils = UnitTestUtils(__file__)
-
-    # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
-
-    # use the local instance of the epidata database
-    secrets.db.host = 'delphi_database_epidata'
-    secrets.db.epi = ('user', 'pass')
-
-    # clear relevant tables
-    with Database.connect() as db:
-      with db.new_cursor() as cur:
-        cur.execute('truncate table covid_hosp_facility')
-        cur.execute('truncate table covid_hosp_facility_key')
-        cur.execute('truncate table covid_hosp_meta')
+  db_class = Database
+  test_util_context = __file__
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):
     """Acquire a new dataset."""
 
-    # only mock out network calls to external hosts
-    mock_network = MagicMock()
-    mock_network.fetch_metadata.return_value = \
-        self.test_utils.load_sample_metadata()
-    mock_network.fetch_dataset.return_value = \
-        self.test_utils.load_sample_dataset()
-
     # make sure the data does not yet exist
     with self.subTest(name='no data yet'):
       response = Epidata.covid_hosp_facility(
           '450822', Epidata.range(20200101, 20210101))
       self.assertEqual(response['result'], -2, response)
 
     # acquire sample data into local database
-    with self.subTest(name='first acquisition'):
-      acquired = Update.run(network=mock_network)
+    with self.subTest(name='first acquisition'), \
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
+         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
+      acquired = Database().update_dataset()
       self.assertTrue(acquired)
 
     # make sure the data now exists
@@ -89,12 +67,14 @@ def test_acquire_dataset(self):
         else:
           self.assertEqual(row[k], v, f"row[{k}] is {row[k]} not {v}")
 
-      # expect 113 fields per row (114 database columns, except `id`)
-      self.assertEqual(len(row), 113)
+      # Expect len(row) to equal the amount of dynamic columns + one extra issue column
+      self.assertEqual(len(row), len(list(CovidHospSomething().columns('covid_hosp_facility'))) + 1)
 
     # re-acquisition of the same dataset should be a no-op
-    with self.subTest(name='second acquisition'):
-      acquired = Update.run(network=mock_network)
+    with self.subTest(name='second acquisition'), \
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
+         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
+      acquired = Database().update_dataset()
       self.assertFalse(acquired)
 
     # make sure the data still exists
@@ -108,16 +88,11 @@ def test_acquire_dataset(self):
   def test_facility_lookup(self):
     """Lookup facilities using various filters."""
 
-    # only mock out network calls to external hosts
-    mock_network = MagicMock()
-    mock_network.fetch_metadata.return_value = \
-        self.test_utils.load_sample_metadata()
-    mock_network.fetch_dataset.return_value = \
-        self.test_utils.load_sample_dataset()
-
     # acquire sample data into local database
-    with self.subTest(name='first acquisition'):
-      acquired = Update.run(network=mock_network)
+    with self.subTest(name='first acquisition'), \
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
+         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
+      acquired = Database().update_dataset()
       self.assertTrue(acquired)
 
     # texas ground truth, sorted by `hospital_pk`
@@ -181,16 +156,11 @@ def test_facility_lookup(self):
       response = Epidata.covid_hosp_facility_lookup(state='not a state')
       self.assertEqual(response['result'], -2)
 
-    # update facility info
-    mock_network = MagicMock()
-    mock_network.fetch_metadata.return_value = \
-        self.test_utils.load_sample_metadata('metadata_update_facility.csv')
-    mock_network.fetch_dataset.return_value = \
-        self.test_utils.load_sample_dataset('dataset_update_facility.csv')
-
-    # acquire sample data into local database
-    with self.subTest(name='second acquisition'):
-      acquired = Update.run(network=mock_network)
+    # acquire sample data into local database with updated facility info
+    with self.subTest(name='second acquisition'), \
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata('metadata_update_facility.csv')), \
+         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset('dataset_update_facility.csv')):
+      acquired = Database().update_dataset()
       self.assertTrue(acquired)
 
     texas_hospitals[1]['zip'] = '88888'

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -11,39 +11,20 @@
 
 # first party
 from delphi.epidata.acquisition.covid_hosp.state_daily.database import Database
-from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
+from delphi.epidata.acquisition.covid_hosp.common.test_utils import CovidHospTestCase, UnitTestUtils
 from delphi.epidata.client.delphi_epidata import Epidata
-from delphi.epidata.acquisition.covid_hosp.state_daily.update import Update
-from delphi.epidata.acquisition.covid_hosp.state_daily.network import Network
-from delphi.epidata.acquisition.covid_hosp.common.utils import Utils
-import delphi.operations.secrets as secrets
+from delphi.epidata.acquisition.covid_hosp.common.network import Network
+from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething
 
 # py3tester coverage target (equivalent to `import *`)
 __test_target__ = \
     'delphi.epidata.acquisition.covid_hosp.state_daily.update'
 
 
-class AcquisitionTests(unittest.TestCase):
+class AcquisitionTests(CovidHospTestCase):
 
-  def setUp(self):
-    """Perform per-test setup."""
-
-    # configure test data
-    self.test_utils = UnitTestUtils(__file__)
-
-    # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
-
-    # use the local instance of the epidata database
-    secrets.db.host = 'delphi_database_epidata'
-    secrets.db.epi = ('user', 'pass')
-
-    # clear relevant tables
-    with Database.connect() as db:
-      with db.new_cursor() as cur:
-        cur.execute('truncate table covid_hosp_state_daily')
-        cur.execute('truncate table covid_hosp_state_timeseries')
-        cur.execute('truncate table covid_hosp_meta')
+  db_class = Database
+  test_util_context = __file__
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):
@@ -61,8 +42,8 @@ def test_acquire_dataset(self):
          patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset("dataset0.csv"), # dataset for 3/13
                                                              self.test_utils.load_sample_dataset("dataset0.csv"), # first dataset for 3/15
                                                              self.test_utils.load_sample_dataset()] # second dataset for 3/15
-                      ) as mock_fetch:
-      acquired = Update.run()
+                      ):
+      acquired = Database().update_dataset()
       self.assertTrue(acquired)
       self.assertEqual(mock_fetch_meta.call_count, 1)
 
@@ -82,18 +63,18 @@ def test_acquire_dataset(self):
       self.assertAlmostEqual(actual, expected)
       self.assertIsNone(row['critical_staffing_shortage_today_no'])
 
-      # expect 61 fields per row (62 database columns, except `id`) # TODO: ???  this is wrong!
-      self.assertEqual(len(row), 118)
+      # Expect len(row) to equal the amount of dynamic columns + one extra issue column
+      self.assertEqual(len(row), len(list(CovidHospSomething().columns('state_daily'))) + 1)
 
     with self.subTest(name='all date batches acquired'):
       response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101), issues=20210313)
       self.assertEqual(response['result'], 1)
 
     # re-acquisition of the same dataset should be a no-op
     with self.subTest(name='second acquisition'), \
-         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
-         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()) as mock_fetch:
-      acquired = Update.run()
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
+         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
+      acquired = Database().update_dataset()
       self.assertFalse(acquired)
 
     # make sure the data still exists
@@ -114,18 +95,14 @@ def test_acquire_specific_issue(self):
 
     # acquire sample data into local database
     # mock out network calls to external hosts
-    with Database.connect() as db:
+    with Database().connect() as db:
       pre_max_issue = db.get_max_issue()
     self.assertEqual(pre_max_issue, pd.Timestamp('1900-01-01 00:00:00'))
     with self.subTest(name='first acquisition'), \
-         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()) as mock_fetch_meta, \
-         patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset("dataset0.csv")]
-                      ) as mock_fetch:
-      acquired = Utils.update_dataset(Database,
-                                      Network,
-                                      date(2021, 3, 12),
-                                      date(2021, 3, 14))
-      with Database.connect() as db:
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
+         patch.object(Network, 'fetch_dataset', side_effect=[self.test_utils.load_sample_dataset("dataset0.csv")]):
+      acquired = Database().update_dataset(date(2021, 3, 12), date(2021, 3, 14))
+      with Database().connect() as db:
         post_max_issue = db.get_max_issue()
       self.assertEqual(post_max_issue, pd.Timestamp('2021-03-13 00:00:00'))
       self.assertTrue(acquired)

integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py

@@ -3,13 +3,14 @@
 # standard library
 import unittest
 from unittest.mock import MagicMock
+from unittest.mock import patch
 
 # first party
-from delphi.epidata.acquisition.covid_hosp.common.database import Database
-from delphi.epidata.acquisition.covid_hosp.common.test_utils import UnitTestUtils
+from delphi.epidata.acquisition.covid_hosp.state_timeseries.database import Database
+from delphi.epidata.acquisition.covid_hosp.common.network import Network
+from delphi.epidata.acquisition.covid_hosp.common.test_utils import CovidHospTestCase, UnitTestUtils
 from delphi.epidata.client.delphi_epidata import Epidata
-from delphi.epidata.acquisition.covid_hosp.state_timeseries.update import Update
-import delphi.operations.secrets as secrets
+from delphi.epidata.common.covid_hosp.covid_hosp_schema_io import CovidHospSomething
 
 # third party
 from freezegun import freeze_time
@@ -19,47 +20,28 @@
     'delphi.epidata.acquisition.covid_hosp.state_timeseries.update'
 
 
-class AcquisitionTests(unittest.TestCase):
+class AcquisitionTests(CovidHospTestCase):
 
-  def setUp(self):
-    """Perform per-test setup."""
-
-    # configure test data
-    self.test_utils = UnitTestUtils(__file__)
-
-    # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
-
-    # use the local instance of the epidata database
-    secrets.db.host = 'delphi_database_epidata'
-    secrets.db.epi = ('user', 'pass')
-
-    # clear relevant tables
-    with Database.connect() as db:
-      with db.new_cursor() as cur:
-        cur.execute('truncate table covid_hosp_state_daily')
-        cur.execute('truncate table covid_hosp_state_timeseries')
-        cur.execute('truncate table covid_hosp_meta')
+  db_class = Database
+  test_util_context = __file__
+  # TODO: no need for the following after covid_hosp table split is merged
+  # (in https://github.com/cmu-delphi/delphi-epidata/pull/1126)
+  extra_tables_used = ['covid_hosp_state_daily']
 
   @freeze_time("2021-03-17")
   def test_acquire_dataset(self):
     """Acquire a new dataset."""
 
-    # only mock out network calls to external hosts
-    mock_network = MagicMock()
-    mock_network.fetch_metadata.return_value = \
-        self.test_utils.load_sample_metadata()
-    mock_network.fetch_dataset.return_value = \
-        self.test_utils.load_sample_dataset()
-
     # make sure the data does not yet exist
     with self.subTest(name='no data yet'):
       response = Epidata.covid_hosp('MA', Epidata.range(20200101, 20210101))
       self.assertEqual(response['result'], -2)
 
     # acquire sample data into local database
-    with self.subTest(name='first acquisition'):
-      acquired = Update.run(network=mock_network)
+    with self.subTest(name='first acquisition'), \
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
+         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
+      acquired = Database().update_dataset()
       self.assertTrue(acquired)
 
     # make sure the data now exists
@@ -78,12 +60,14 @@ def test_acquire_dataset(self):
       self.assertAlmostEqual(actual, expected)
       self.assertIsNone(row['critical_staffing_shortage_today_no'])
 
-      # expect 61 fields per row (62 database columns, except `id`) # TODO: ???  this is wrong!
-      self.assertEqual(len(row), 118)
+      # Expect len(row) to equal the amount of dynamic columns + one extra issue column
+      self.assertEqual(len(row), len(list(CovidHospSomething().columns('state_timeseries'))) + 1)
 
     # re-acquisition of the same dataset should be a no-op
-    with self.subTest(name='second acquisition'):
-      acquired = Update.run(network=mock_network)
+    with self.subTest(name='second acquisition'), \
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata()), \
+         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset()):
+      acquired = Database().update_dataset()
       self.assertFalse(acquired)
 
     # make sure the data still exists
@@ -93,13 +77,11 @@ def test_acquire_dataset(self):
       self.assertEqual(len(response['epidata']), 1)
 
     # acquire new data into local database
-    with self.subTest(name='first acquisition'):
+    with self.subTest(name='updated acquisition'), \
+         patch.object(Network, 'fetch_metadata', return_value=self.test_utils.load_sample_metadata("metadata2.csv")), \
+         patch.object(Network, 'fetch_dataset', return_value=self.test_utils.load_sample_dataset("dataset2.csv")):
       # acquire new data with 3/16 issue date
-      mock_network.fetch_metadata.return_value = \
-        self.test_utils.load_sample_metadata("metadata2.csv")
-      mock_network.fetch_dataset.return_value = \
-        self.test_utils.load_sample_dataset("dataset2.csv")
-      acquired = Update.run(network=mock_network)
+      acquired = Database().update_dataset()
       self.assertTrue(acquired)
 
     with self.subTest(name='as_of checks'):