Update sql table definitions and add extra na values to historic data script

Author: cchuong

src/acquisition/rvdss/pull_historic.py

@@ -414,7 +414,8 @@ def fetch_one_season_from_report(url):
 
             # Read table, coding all the abbreviations for missing data into NA
             # Also use dropna because removing footers causes the html to have an empty row
-            na_values = ['N.A.','N.A', 'N.C.','N.R.','Not Available','Not Tested',"not available","not tested","N.D.","-"]
+            na_values = ['N.A.','N.A', 'N.C.','N.R.','Not Available','Not Tested',"not available",
+                         "not tested","N.D.","-",'Not tested','non testé']
             table =  pd.read_html(tab,na_values=na_values)[0].dropna(how="all")
 
             # Check for multiline headers

src/ddl/rvdss.sql

@@ -5,45 +5,107 @@ TODO: briefly describe data source and define all columns.
 
 CREATE TABLE `rvdss_repiratory_detections` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
-  `date` date NOT NULL,
-  `geo_type` char(20) NOT NULL,
-  `geo_value` char(20) NOT NULL,
-  `epiweek` int(11) NOT NULL,
-  `flua_positive_tests` int(11) NOT NULL,
-  `flua_percent_positive_tests` double NOT NULL,
-  `flu_total_tests` int(11) NOT NULL,
+  `epiweek` int(6) NOT NULL,
+  `time_value` date NOT NULL,
+  `issue` date NOT NULL,
+  `geo_type` char(6) NOT NULL,
+  `geo_value` char(35) NOT NULL,
+  `sarscov2_tests` int(10) NOT NULL,
+  `sarscov2_positive_tests` int(10) NOT NULL,
+  `flu_tests` int(10) NOT NULL,
+  `flu_positive_tests` int(10) NOT NULL,
+  `fluah1n1pdm09_positive_tests` int(10) NOT NULL,
+  `fluah3_positive_tests` int(10) NOT NULL,
+  `fluauns_positive_tests` int(10) NOT NULL,
+  `flua_positive_tests` int(10) NOT NULL,
+  `flub_positive_tests` int(10) NOT NULL,
+  `rsv_tests` int(10) NOT NULL,
+  `rsv_positive_tests` int(10) NOT NULL,
+  `hpiv_tests` int(10) NOT NULL,
+  `hpiv1_positive_tests` int(10) NOT NULL,
+  `hpiv2_positive_tests` int(10) NOT NULL,
+  `hpiv3_positive_tests` int(10) NOT NULL,
+  `hpiv4_positive_tests` int(10) NOT NULL,
+  `hpivother_positive_tests` int(10) NOT NULL,
+  `adv_tests` int(10) NOT NULL,
+  `adv_positive_tests` int(10) NOT NULL,
+  `hmpv_tests` int(10) NOT NULL,
+  `hmpv_positive_tests` int(10) NOT NULL,
+  `evrv_tests` int(10) NOT NULL,
+  `evrv_positive_tests` int(10) NOT NULL,
+  `hcov_tests` int(10) NOT NULL,
+  `hcov_positive_tests` int(10) NOT NULL,
+  `week` int(2) NOT NULL,
+  `weekorder` int(2) NOT NULL,
+  `year` int(4) NOT NULL,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `date` (`date`,`geo_value`),
+  UNIQUE KEY `date` (`epiweek`, `time_value`,`issue`, `geo_type`,`geo_value`),
   KEY `state` (`state`),
   KEY `epiweek` (`epiweek`),
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-CREATE TABLE `rvdss_testing` (
+CREATE TABLE `rvdss_pct_positive` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
-  `date` date NOT NULL,
-  `geo_type` char(20) NOT NULL,
-  `geo_value` char(20) NOT NULL,
-  `epiweek` int(11) NOT NULL,
-  `flua_positive_tests` int(11) NOT NULL,
-  `flua_percent_positive_tests` double NOT NULL,
-  `flu_total_tests` int(11) NOT NULL,
+  `epiweek` int(6) NOT NULL,
+  `time_value` date NOT NULL,
+  `issue` date NOT NULL,
+  `geo_type` char(6) NOT NULL,
+  `geo_value` char(35) NOT NULL,
+  `evrv_pct_positive` int(10) NOT NULL,
+  `evrv_tests` int(10) NOT NULL,
+  `evrv_positive_tests` int(10) NOT NULL,
+  `hpiv_pct_positive` int(10) NOT NULL,
+  `hpiv_tests` int(10) NOT NULL,
+  `hpiv_positive_tests` int(10) NOT NULL,
+  `adv_pct_positive` int(10) NOT NULL,
+  `adv_tests` int(10) NOT NULL,
+  `hcov_pct_positive` int(10) NOT NULL,
+  `hcov_tests` int(10) NOT NULL,
+  `hcov_positive_tests` int(10) NOT NULL,
+  `flua_pct_positive` int(10) NOT NULL,
+  `flub_pct_positive` int(10) NOT NULL,
+  `flu_tests` int(10) NOT NULL,
+  `flua_positive_tests` int(10) NOT NULL,
+  `flua_tests` int(10) NOT NULL,
+  `flub_tests` int(10) NOT NULL,
+  `flub_positive_tests` int(10) NOT NULL,
+  `flu_positive_tests` int(10) NOT NULL,
+  `flu_pct_positive` int(10) NOT NULL,
+  `hmpv_pct_positive` int(10) NOT NULL,
+  `hmpv_tests` int(10) NOT NULL,
+  `hmpv_positive_tests` int(10) NOT NULL,
+  `rsv_pct_positive` int(10) NOT NULL,
+  `rsv_tests` int(10) NOT NULL,
+  `rsv_positive_tests` int(10) NOT NULL,
+  `sarscov2_pct_positive` int(10) NOT NULL,
+  `sarscov2_tests` int(10) NOT NULL,
+  `sarscov2_positive_tests` int(10) NOT NULL,
+  `region` char(20)
+  `week` int(2) NOT NULL,
+  `weekorder` int(2) NOT NULL,
+  `year` int(4) NOT NULL,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `date` (`date`,`geo_value`),
+  UNIQUE KEY `date` (`epiweek`, `time_value`,`issue`, `geo_type`,`geo_value`),
   KEY `state` (`state`),
   KEY `epiweek` (`epiweek`),
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 CREATE TABLE `rvdss_detections_counts` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
-  `date` date NOT NULL,
-  `geo_type` char(20) NOT NULL,
-  `geo_value` char(20) NOT NULL,
-  `epiweek` int(11) NOT NULL,
-  `flua_positive_tests` int(11) NOT NULL,
-  `flua_percent_positive_tests` double NOT NULL,
-  `flu_total_tests` int(11) NOT NULL,
+  `epiweek` int(6) NOT NULL,
+  `time_value` date NOT NULL,
+  `issue` date NOT NULL,
+  `geo_type` char(6) NOT NULL,
+  `geo_value` char(35) NOT NULL,
+  `hpiv_positive_tests` int(10) NOT NULL,
+  `adv_positive_tests` int(10) NOT NULL,
+  `hmpv_positive_tests` int(10) NOT NULL,
+  `evrv_positive_tests` int(10) NOT NULL,
+  `hcov_positive_tests` int(10) NOT NULL,
+  `rsv_positive_tests` int(10) NOT NULL,
+  `flu_positive_tests` int(10) NOT NULL,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `date` (`date`,`geo_value`),
+  UNIQUE KEY `date` (`epiweek`, `time_value`,`issue`, `geo_type`,`geo_value`),
   KEY `state` (`state`),
   KEY `epiweek` (`epiweek`),
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;