mirror of
https://github.com/Haiti-Distributed-Healthcare-System/hdhs.git
synced 2026-10-26 17:35:23 +00:00
Merge pull request #101 from Haiti-Distributed-Healthcare-System/feature/data_validation_t77
Postgres Data Validation
This commit is contained in:
8 files changed
+3916
-2
No files matched your search
Whitespace-only changes.
+14
-2
@@ -1,12 +1,14 @@
|
||||
import pandas as pd
|
||||
from sqlalchemy import create_engine
|
||||
import sqlalchemy as database
|
||||
|
||||
from db import DB
|
||||
|
||||
|
||||
class PSQL(DB):
|
||||
def __init__(self, db_uri):
|
||||
self.__engine = create_engine(db_uri)
|
||||
self.__engine = database.create_engine(db_uri)
|
||||
self.__table_cache = {}
|
||||
self.__metadata = database.MetaData()
|
||||
|
||||
# METHODS FROM DB SUPER CLASS
|
||||
# ###########################
|
||||
@@ -24,6 +26,16 @@ class PSQL(DB):
|
||||
|
||||
# ###########################
|
||||
|
||||
@property
|
||||
def db(self,):
|
||||
return database
|
||||
|
||||
def table(self, table_name):
|
||||
if table_name not in self.__table_cache.keys():
|
||||
self.__table_cache[table_name] = self.db.Table(table_name, self.__metadata, autoload=True, autoload_with=self.__engine)
|
||||
|
||||
return self.__table_cache[table_name]
|
||||
|
||||
def open_connection(self):
|
||||
return self.__engine.connect()
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
|
||||
# This script will go through and validate that patient ages increase across visits (where visits are ordered chronologically)
|
||||
|
||||
### pseudocode
|
||||
# get unique patient_ids
|
||||
# for each unique patient_id
|
||||
# get all visits from patient (ordered by visit date)
|
||||
# get all visits from patient (ordered by visit age, then by date within the same age)
|
||||
# assert that these visit orderings are equal
|
||||
#
|
||||
|
||||
from psql import PSQL
|
||||
import logging
|
||||
import sys
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
logging.basicConfig(filename=f"./validation/{datetime.now().strftime('%Y%m%d_%H%M%S')}_sanity_age_increases_across_visits.log", level=logging.INFO)
|
||||
logging.info('Starting log to validate that age increases across chronological visits')
|
||||
|
||||
|
||||
def main():
|
||||
psql = PSQL("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db")
|
||||
|
||||
visits = psql.table('visits')
|
||||
|
||||
ids_dates_ages_query = psql.db.select([visits.columns.pk_visit_id, visits.columns.fk_patient_id, visits.columns.visit_date, visits.columns.visit_age])
|
||||
|
||||
with psql.open_connection() as connection:
|
||||
ids_dates_ages = connection.execute(ids_dates_ages_query).fetchall()
|
||||
|
||||
df = pd.DataFrame(ids_dates_ages, columns=['pk_visit_id', 'fk_patient_id', 'visit_date', 'visit_age'])
|
||||
df['visit_date'] = pd.to_datetime(df['visit_date'])
|
||||
patient_ids = df['fk_patient_id'].unique()
|
||||
|
||||
for patient_id in patient_ids:
|
||||
visit_dates_and_ages = df.loc[df['fk_patient_id'] == patient_id]
|
||||
|
||||
# Check for and log null age entries
|
||||
check_null_ages = visit_dates_and_ages.query('visit_age != visit_age')
|
||||
if len(check_null_ages.index) > 0:
|
||||
logging.info(f"patient ({patient_id}) does not have a visit age on visit(s) ({check_null_ages['pk_visit_id'].values.tolist()})")
|
||||
|
||||
# Check for and log null date entries
|
||||
check_null_dates = visit_dates_and_ages.query('visit_date != visit_date')
|
||||
if len(check_null_dates.index) > 0:
|
||||
logging.info(f"patient ({patient_id}) does not have a visit date on visit(s) ({check_null_dates['pk_visit_id'].values.tolist()})")
|
||||
|
||||
visit_dates_and_ages = visit_dates_and_ages.dropna()
|
||||
visits_by_age = visit_dates_and_ages.sort_values(by=['visit_age', 'visit_date'])
|
||||
visits_by_date = visit_dates_and_ages.sort_values(by=['visit_date'])
|
||||
|
||||
try:
|
||||
pd.testing.assert_frame_equal(visits_by_age, visits_by_date)
|
||||
except AssertionError:
|
||||
logging.info(f"patient ({patient_id}) has their age go down across visits ({visits_by_date['pk_visit_id'].values.tolist()})")
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
|
||||
# This script will go through and validate that all patients listed as pregnent are female
|
||||
|
||||
### pseudocode
|
||||
# for each visit in the visits table
|
||||
# if visit indicates pregnancy
|
||||
# assert the corresponding patient is female
|
||||
#
|
||||
|
||||
from psql import PSQL
|
||||
import logging
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
logging.basicConfig(filename=f"./validation/{datetime.now().strftime('%Y%m%d_%H%M%S')}_sanity_pregnant_implies_female.log", level=logging.INFO)
|
||||
logging.info('Starting log to validate pregnant patients are female')
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
psql = PSQL("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db")
|
||||
|
||||
visits = psql.table('visits')
|
||||
patients = psql.table('patients')
|
||||
|
||||
pregnant_visits_query = psql.db.select([visits]).where(visits.columns.ob_pregnant == 'Y')
|
||||
|
||||
with psql.open_connection() as connection:
|
||||
pregnant_visits = connection.execute(pregnant_visits_query).fetchall()
|
||||
|
||||
for visit in pregnant_visits:
|
||||
select_patient_query = psql.db.select([patients.columns.gender]).where(patients.columns.pk_patient_id == visit.fk_patient_id)
|
||||
|
||||
patient_record = connection.execute(select_patient_query).fetchall()
|
||||
|
||||
try:
|
||||
assert patient_record[0][0] == 'F' # attempt to assert the patient is female
|
||||
except AssertionError as assertion_error:
|
||||
logging.info(f"patient ({visit.fk_patient_id}) w/ gender ({patient_record[0][0]}) is listed as pregnant on visit ({visit.pk_visit_id})")
|
||||
except IndexError as index_error:
|
||||
logging.warning(f"patient ({visit.fk_patient_id}) was not found in the patients table")
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
|
||||
Whitespace-only changes.
@@ -0,0 +1,109 @@
|
||||
INFO:root:Starting log to validate pregnant patients are female
|
||||
INFO:root:patient (19) w/ gender (M) is listed as pregnant on visit (20)
|
||||
INFO:root:patient (228) w/ gender (M) is listed as pregnant on visit (254)
|
||||
INFO:root:patient (718) w/ gender (None) is listed as pregnant on visit (776)
|
||||
INFO:root:patient (758) w/ gender (None) is listed as pregnant on visit (818)
|
||||
INFO:root:patient (2606) w/ gender (None) is listed as pregnant on visit (2956)
|
||||
INFO:root:patient (2621) w/ gender (None) is listed as pregnant on visit (2971)
|
||||
INFO:root:patient (2641) w/ gender (M) is listed as pregnant on visit (2991)
|
||||
INFO:root:patient (5641) w/ gender (M) is listed as pregnant on visit (5666)
|
||||
INFO:root:patient (5772) w/ gender (M) is listed as pregnant on visit (5811)
|
||||
INFO:root:patient (6095) w/ gender (M) is listed as pregnant on visit (6205)
|
||||
INFO:root:patient (9327) w/ gender (M) is listed as pregnant on visit (6464)
|
||||
INFO:root:patient (7072) w/ gender (None) is listed as pregnant on visit (7352)
|
||||
INFO:root:patient (8806) w/ gender (None) is listed as pregnant on visit (8866)
|
||||
INFO:root:patient (8870) w/ gender (None) is listed as pregnant on visit (8946)
|
||||
INFO:root:patient (8945) w/ gender (None) is listed as pregnant on visit (9041)
|
||||
WARNING:root:patient (9048125) was not found in the patients table
|
||||
INFO:root:patient (9851) w/ gender (None) is listed as pregnant on visit (9956)
|
||||
INFO:root:patient (9622) w/ gender (M) is listed as pregnant on visit (9618)
|
||||
INFO:root:patient (9649) w/ gender (None) is listed as pregnant on visit (9649)
|
||||
INFO:root:patient (9653) w/ gender (M) is listed as pregnant on visit (9653)
|
||||
INFO:root:patient (11306) w/ gender (M) is listed as pregnant on visit (12098)
|
||||
INFO:root:patient (12814) w/ gender (None) is listed as pregnant on visit (14400)
|
||||
INFO:root:patient (12832) w/ gender (None) is listed as pregnant on visit (14418)
|
||||
INFO:root:patient (15037) w/ gender (None) is listed as pregnant on visit (18069)
|
||||
INFO:root:patient (15083) w/ gender (None) is listed as pregnant on visit (18147)
|
||||
INFO:root:patient (15401) w/ gender (M) is listed as pregnant on visit (18663)
|
||||
INFO:root:patient (15405) w/ gender (M) is listed as pregnant on visit (18671)
|
||||
INFO:root:patient (16313) w/ gender (M) is listed as pregnant on visit (20273)
|
||||
INFO:root:patient (17622) w/ gender (M) is listed as pregnant on visit (22835)
|
||||
INFO:root:patient (17945) w/ gender (M) is listed as pregnant on visit (23555)
|
||||
INFO:root:patient (18334) w/ gender (None) is listed as pregnant on visit (24394)
|
||||
INFO:root:patient (18424) w/ gender (None) is listed as pregnant on visit (24599)
|
||||
INFO:root:patient (18426) w/ gender (None) is listed as pregnant on visit (24600)
|
||||
INFO:root:patient (18593) w/ gender (None) is listed as pregnant on visit (24909)
|
||||
INFO:root:patient (18652) w/ gender (None) is listed as pregnant on visit (24985)
|
||||
INFO:root:patient (18656) w/ gender (None) is listed as pregnant on visit (24989)
|
||||
INFO:root:patient (18659) w/ gender (None) is listed as pregnant on visit (25001)
|
||||
INFO:root:patient (13657) w/ gender (M) is listed as pregnant on visit (25023)
|
||||
INFO:root:patient (17970) w/ gender (M) is listed as pregnant on visit (25183)
|
||||
INFO:root:patient (19355) w/ gender (M) is listed as pregnant on visit (26403)
|
||||
INFO:root:patient (29062) w/ gender (nan) is listed as pregnant on visit (27740)
|
||||
INFO:root:patient (18678) w/ gender (M) is listed as pregnant on visit (29055)
|
||||
INFO:root:patient (20948) w/ gender (None) is listed as pregnant on visit (29442)
|
||||
INFO:root:patient (20162) w/ gender (None) is listed as pregnant on visit (30680)
|
||||
INFO:root:patient (21745) w/ gender (None) is listed as pregnant on visit (30831)
|
||||
INFO:root:patient (22390) w/ gender (M) is listed as pregnant on visit (32147)
|
||||
INFO:root:patient (22515) w/ gender (None) is listed as pregnant on visit (32317)
|
||||
INFO:root:patient (19564) w/ gender (M) is listed as pregnant on visit (32925)
|
||||
INFO:root:patient (20433) w/ gender (M) is listed as pregnant on visit (32965)
|
||||
INFO:root:patient (23178) w/ gender (None) is listed as pregnant on visit (33513)
|
||||
INFO:root:patient (23222) w/ gender (None) is listed as pregnant on visit (33577)
|
||||
INFO:root:patient (23814) w/ gender (M) is listed as pregnant on visit (34757)
|
||||
INFO:root:patient (24021) w/ gender (None) is listed as pregnant on visit (35088)
|
||||
INFO:root:patient (23961) w/ gender (None) is listed as pregnant on visit (36205)
|
||||
INFO:root:patient (24797) w/ gender (None) is listed as pregnant on visit (36419)
|
||||
INFO:root:patient (24840) w/ gender (None) is listed as pregnant on visit (36486)
|
||||
INFO:root:patient (24982) w/ gender (None) is listed as pregnant on visit (36820)
|
||||
INFO:root:patient (8774) w/ gender (M) is listed as pregnant on visit (36852)
|
||||
INFO:root:patient (25417) w/ gender (None) is listed as pregnant on visit (37694)
|
||||
INFO:root:patient (25418) w/ gender (None) is listed as pregnant on visit (37696)
|
||||
INFO:root:patient (25439) w/ gender (None) is listed as pregnant on visit (37729)
|
||||
INFO:root:patient (25647) w/ gender (None) is listed as pregnant on visit (38020)
|
||||
INFO:root:patient (25697) w/ gender (None) is listed as pregnant on visit (38121)
|
||||
INFO:root:patient (25734) w/ gender (None) is listed as pregnant on visit (38191)
|
||||
INFO:root:patient (26546) w/ gender (None) is listed as pregnant on visit (38413)
|
||||
INFO:root:patient (26295) w/ gender (M) is listed as pregnant on visit (39233)
|
||||
INFO:root:patient (26540) w/ gender (M) is listed as pregnant on visit (39676)
|
||||
INFO:root:patient (25647) w/ gender (None) is listed as pregnant on visit (39686)
|
||||
INFO:root:patient (26540) w/ gender (M) is listed as pregnant on visit (40698)
|
||||
INFO:root:patient (27143) w/ gender (M) is listed as pregnant on visit (41071)
|
||||
INFO:root:patient (27236) w/ gender (None) is listed as pregnant on visit (41192)
|
||||
INFO:root:patient (27262) w/ gender (None) is listed as pregnant on visit (41227)
|
||||
INFO:root:patient (27563) w/ gender (None) is listed as pregnant on visit (41838)
|
||||
INFO:root:patient (27734) w/ gender (None) is listed as pregnant on visit (42126)
|
||||
INFO:root:patient (27776) w/ gender (None) is listed as pregnant on visit (42236)
|
||||
INFO:root:patient (28105) w/ gender (None) is listed as pregnant on visit (42885)
|
||||
INFO:root:patient (28112) w/ gender (None) is listed as pregnant on visit (42891)
|
||||
INFO:root:patient (28164) w/ gender (None) is listed as pregnant on visit (42968)
|
||||
INFO:root:patient (28170) w/ gender (None) is listed as pregnant on visit (42978)
|
||||
INFO:root:patient (26540) w/ gender (M) is listed as pregnant on visit (43322)
|
||||
INFO:root:patient (28564) w/ gender (None) is listed as pregnant on visit (43876)
|
||||
INFO:root:patient (28701) w/ gender (None) is listed as pregnant on visit (44127)
|
||||
INFO:root:patient (28744) w/ gender (None) is listed as pregnant on visit (44215)
|
||||
INFO:root:patient (28961) w/ gender (None) is listed as pregnant on visit (44616)
|
||||
INFO:root:patient (29629) w/ gender (None) is listed as pregnant on visit (46000)
|
||||
INFO:root:patient (29656) w/ gender (None) is listed as pregnant on visit (46042)
|
||||
INFO:root:patient (29681) w/ gender (None) is listed as pregnant on visit (46080)
|
||||
INFO:root:patient (29690) w/ gender (None) is listed as pregnant on visit (46095)
|
||||
INFO:root:patient (28744) w/ gender (None) is listed as pregnant on visit (46309)
|
||||
INFO:root:patient (25788) w/ gender (None) is listed as pregnant on visit (46396)
|
||||
INFO:root:patient (28701) w/ gender (None) is listed as pregnant on visit (46457)
|
||||
INFO:root:patient (30123) w/ gender (None) is listed as pregnant on visit (46957)
|
||||
INFO:root:patient (30281) w/ gender (None) is listed as pregnant on visit (47372)
|
||||
INFO:root:patient (28924) w/ gender (M) is listed as pregnant on visit (49108)
|
||||
INFO:root:patient (31203) w/ gender (None) is listed as pregnant on visit (49281)
|
||||
INFO:root:patient (31461) w/ gender (None) is listed as pregnant on visit (50149)
|
||||
INFO:root:patient (31470) w/ gender (None) is listed as pregnant on visit (50205)
|
||||
INFO:root:patient (31813) w/ gender (None) is listed as pregnant on visit (50241)
|
||||
INFO:root:patient (31482) w/ gender (None) is listed as pregnant on visit (50244)
|
||||
INFO:root:patient (31534) w/ gender (M) is listed as pregnant on visit (50425)
|
||||
INFO:root:patient (31594) w/ gender (None) is listed as pregnant on visit (50615)
|
||||
INFO:root:patient (31726) w/ gender (None) is listed as pregnant on visit (50963)
|
||||
INFO:root:patient (31724) w/ gender (None) is listed as pregnant on visit (50964)
|
||||
INFO:root:patient (31771) w/ gender (None) is listed as pregnant on visit (51064)
|
||||
INFO:root:patient (31899) w/ gender (None) is listed as pregnant on visit (51313)
|
||||
INFO:root:patient (32112) w/ gender (M) is listed as pregnant on visit (51872)
|
||||
INFO:root:patient (32168) w/ gender (None) is listed as pregnant on visit (52003)
|
||||
INFO:root:patient (32268) w/ gender (None) is listed as pregnant on visit (52222)
|
||||
File diff suppressed because it is too large.
Load diff
Whitespace-only changes.
Reference in new issue
Block a user