diff --git a/.gitignore b/.gitignore index 789dd37..69bc636 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ **/.lighthouseci/** - +env/* +.vscode/* # postgres persisted data pgdata/ + +.gitsecret/keys/random_seed +!*.secret diff --git a/.gitsecret/keys/pubring.kbx b/.gitsecret/keys/pubring.kbx new file mode 100644 index 0000000..47ce7c0 Binary files /dev/null and b/.gitsecret/keys/pubring.kbx differ diff --git a/.gitsecret/keys/pubring.kbx~ b/.gitsecret/keys/pubring.kbx~ new file mode 100644 index 0000000..67af14b Binary files /dev/null and b/.gitsecret/keys/pubring.kbx~ differ diff --git a/.gitsecret/keys/trustdb.gpg b/.gitsecret/keys/trustdb.gpg new file mode 100644 index 0000000..90e4459 Binary files /dev/null and b/.gitsecret/keys/trustdb.gpg differ diff --git a/.gitsecret/paths/mapping.cfg b/.gitsecret/paths/mapping.cfg new file mode 100644 index 0000000..e69de29 diff --git a/db/.gitignore b/db/.gitignore new file mode 100644 index 0000000..59ee0a5 --- /dev/null +++ b/db/.gitignore @@ -0,0 +1 @@ +**/__pycache__/** diff --git a/db/README.md b/db/README.md new file mode 100644 index 0000000..820f0d3 --- /dev/null +++ b/db/README.md @@ -0,0 +1,29 @@ +# Database Management + +How to load a backup into your postgres docker container + +First, decrypt the pg_dump files. +``` +git secret reveal +``` +Second, restore the pg_dump into the container. +```bash +gunzip -c ./backups/version.sql.gz | docker exec -i hdhs_database_1 psql -U postgres -d app_db +``` + +## Running Notes + +generate backups: +```bash +docker exec hdhs_database_1 pg_dump -U postgres app_db | gzip > backups/20200408.tar.gz +``` + +Enter database container +```bash +docker-compose -f dev-docker-compose.yml run database bash +``` + +Enter DB inside container +```bash +psql --host=database --username=postgres --dbname=app_db +``` \ No newline at end of file diff --git a/db/clean-python.bash b/db/clean-python.bash new file mode 100755 index 0000000..5e65ecd --- /dev/null +++ b/db/clean-python.bash @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Author: "abpwrs" +# Date: 20200409 + +# script: +isort -rc . +black . + diff --git a/db/db.py b/db/db.py new file mode 100644 index 0000000..51056fa --- /dev/null +++ b/db/db.py @@ -0,0 +1,71 @@ +from abc import ABC, abstractmethod + +import numpy as np +import pandas as pd + +from utils import edit_distance + + +""" +DESC: + Abstract Base Class (ABC) for a Database Object + Extracts shared logic from the FMP and PSQL classes +""" + + +class DB(ABC): + def __init__(self): + self._model = None + self.datetime_format = "%Y%m%d_%H%M" + self._table_names = [] + + # ABSTRACT METHDOS TO BE IMPLEMENTED BY CHILD CLASSES + # ################################################### + + @abstractmethod + def read_table(self, table_name) -> pd.DataFrame: + pass + + @abstractmethod + def write_table(self, data: pd.DataFrame, table_name): + pass + + # ################################################### + + @property + def table_names(self) -> list: + return self._table_names + + def get_tablename_from_fieldname(self, field_name: str): + try: + assert field_name[:3] == "pk_" or field_name[:3] == "fk_" + except AssertionError as e: + print( + "Can only get meaningful table names from field names on primary or foreign keys" + ) + raise (e) + + # remove "pk_" or "fk_" --> we only enter this function if this has already matched + extracted_field_to_match = field_name[3:] + + # not all fields that start with "pk_" or "fk_" end in "_id" + if field_name[-3:] == "_id": + extracted_field_to_match = extracted_field_to_match[:-3] + else: + print(f"consider fixing fieldname: {field_name}") + + # function that returns the edit distance of the table to the extracted field name + matcher = lambda table_name: edit_distance(table_name, extracted_field_to_match) + + try: + # map lambda over list of table `names & extract min edit distance + index_of_match = int( + np.argmin(np.array(list(map(matcher, self.table_names))), axis=0) + ) + except TypeError as e: + print("FAILURE: two tables have equally close edit_distance") + # TODO: potentially add a backup lookup table here + # for now we will re-raise the error and crash the program + raise (e) + + return self.table_names[index_of_match] diff --git a/db/execute-sql-in-container.py b/db/execute-sql-in-container.py new file mode 100755 index 0000000..73a0181 --- /dev/null +++ b/db/execute-sql-in-container.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# Author: "hertz" +# Date: 20200310 +# #################### +# DESC: +# A script to execute a SQL script inside of postgres docker container +# #################### + + +# args: +# 1 -- name of container running postgres to execute the script inside of +# 2 -- path to .SQL script to be execute inside of the postgres container + + +import argparse +import pickle +import sys + +from sqlalchemy import create_engine + +from psql import PSQL + +engine = create_engine("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("command", help="SQL command to run in container") + return parser.parse_args() + + +def main(argv): + # TODO: extract this from database.env (ideally but it in the PSQL class) + psql = PSQL("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db") + result = psql.execute_command(argv.command) + print(result) + + +if __name__ == "__main__": + sys.exit(main(parse_args())) diff --git a/db/fmp.py b/db/fmp.py new file mode 100644 index 0000000..8f0e540 --- /dev/null +++ b/db/fmp.py @@ -0,0 +1,168 @@ +import json +from datetime import datetime + +import numpy as np +import pandas as pd +import pyodbc +from tqdm import tqdm + +from db import DB +from utils import edit_distance + + +""" +File Maker Pro Connection Object + +Desc: +This file contains the primary logic for interacting with the file maker pro database +""" + + +class FMP(DB): + def __init__(self, dsn=None, username=None, password=None, connect=True): + DB.__init__(self) + if dsn is None: + dsn = input("Enter the FMP DSN: ") + if username is None: + username = input("Enter your FMP username: ") + if password is None: + password = input("Enter your FMP password: ") + + self.__dsn = dsn + self.__username = username + self.__password = password + + self.__connection_string = "DSN={0};UID={1};PWD={2}".format( + self.__dsn, self.__username, self.__password + ) + + # TODO: consider extracting this list to the DB class + self._table_names = [ + "globals", + "clinics", + "patients", + "visits", + "diagnoses", + "diagnoses_made", + "meds", + "meds_dispensed", + "geonames", + "settings", + ] + + self._join_tables = [ + "diagnoses_made", + "meds_dispensed" + ] + + self.__python_types_to_sql = { + "str": "VARCHAR", + "date": "TIMESTAMP", + "float64": "FLOAT", + "NoneType": "VARCHAR", + } + + self.__pyodbc_connection = None + self.__cursor = None + if connect: + self.__connect() + + # METHODS FROM DB SUPER CLASS + # ########################### + + def read_table(self, table_name) -> pd.DataFrame: + query = "select * from {table}".format(table=table_name) + df = pd.read_sql(query, self.connection) + return df + + def write_table(self, data, table_name): + # TODO: implement write_table in FMP class + print("IMPLEMENT WRITE_TABLE IN FMP") + + # ########################### + + def __connect(self): + if self.__pyodbc_connection is None: + self.__pyodbc_connection = pyodbc.connect(self.__connection_string) + self.__pyodbc_connection + + def execute_command(self, command_string): + result = None + try: + result = pd.read_sql(command_string, self.connection) + except Exception as e: + print(f"Failure to execute: ['{command_string}']") + raise (e) # TODO: determine if crashing is appropriate + return result + + @property + def python_types_to_sql(self): + return self.__python_types_to_sql + + @property + def connection(self): + if self.__pyodbc_connection is None: + self.__connect() + return self.__pyodbc_connection + + @property + def cursor(self): + if self.__cursor is None: + self.__cursor = self.connection.cursor() + return self.__cursor + + # ABSTRACT BASE CLASS MEHTODS + def parse_database(self) -> dict(): + self._model = dict() + + for table_name in tqdm(self.table_names): + df = self.read_table(table_name) + + self._model[table_name] = {} + col_types = [ + df.iloc[0, index].__class__.__name__ for index in range(len(df.columns)) + ] + + smallest_pk_edit_distance = float("inf") + previous_pk_field = None + for field_name, field_type in zip(df.columns, col_types): + self._model[table_name][field_name] = {} + self._model[table_name][field_name]["type"] = field_type + + # extract primary keys + if field_name[:3] == "pk_": + # possible primary key + distance = edit_distance(field_name[3:-3], table_name) + if distance < smallest_pk_edit_distance: + # closest primary key so far + if previous_pk_field is not None: + # remove previously assumed pk + self._model[table_name][previous_pk_field]["pk"] = False + + # set current pk + self._model[table_name][field_name]["pk"] = True + previous_pk_field = field_name + smallest_pk_edit_distance = distance + else: + self._model[table_name][field_name]["pk"] = False + else: + self._model[table_name][field_name]["pk"] = False + + # extract foreign keys + if field_name[:3] == "fk_": + self._model[table_name][field_name]["fk"] = True + self._model[table_name][field_name][ + "fk_table" + ] = self.get_tablename_from_fieldname(field_name) + else: + self._model[table_name][field_name]["fk"] = False + + return self._model + + def write_model(self, model_object: dict = None): + curr_time = datetime.now().strftime(self.datetime_format) + with open("./models/{time}_model.json".format(time=curr_time), "w") as f: + if model_object is not None: + json.dump(model_object, f, indent=4) + else: + json.dump(self._model, f, indent=4) diff --git a/db/fmp_to_postgres.py b/db/fmp_to_postgres.py new file mode 100644 index 0000000..1a2353b --- /dev/null +++ b/db/fmp_to_postgres.py @@ -0,0 +1,148 @@ +import argparse +import os +import pickle +import sys + +import pandas as pd +import numpy as np +from tqdm import tqdm + +from fmp import FMP +from psql import PSQL +from utils import edit_distance + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--dsn", help="dsn name") + parser.add_argument("-u", "--username", help="username for dsn") + parser.add_argument("-p", "--password", help="password for dsn") + return parser.parse_args() + + +def get_pk_and_fk_from_table(fmp: FMP, df: pd.DataFrame, table_name: str): + pk = None + fks = [] + smallest_pk_edit_distance = float("inf") + for field_name in df.columns: + # extract primary keys + if field_name[:3] == "pk_" and field_name[-3:] == "_id": + # possible primary key + distance = edit_distance(field_name[3:-3], table_name) + if distance < smallest_pk_edit_distance: + pk = field_name + smallest_pk_edit_distance = distance + + # extract foreign keys + if field_name[:3] == "fk_" and field_name[-3:] == "_id": + fks.append((field_name, fmp.get_tablename_from_fieldname(field_name))) + + return pk, fks + + +def save_keys(filename: str, all_data: dict): + np.save(filename, all_data) + + +def load_dict(filename: str): + np_load_old = np.load + np.load = lambda *a, **k,: np_load_old(*a, allow_pickle=True, **k) + np_data = np.load(filename)[()] + + # Restore np.load + np.load = np_load_old + + return np_data + +def main(argv): + fmp = FMP(argv.dsn, argv.username, argv.password) + # TODO: extract username, password, and database name from the database.env file + # TODO: encrypt the database.env file using git-secret + psql = PSQL("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db") + + all_data = dict() + for table_name in tqdm(fmp.table_names): + print(table_name) + df = fmp.read_table(table_name) + # drop all exact duplicates + df.drop_duplicates(inplace=True) + if table_name == "meds_dispensed": + new_df = df[df["pk_meds_dispensed"].notna()] + new_df = new_df[new_df["fk_visit_id"].notna()] + new_df = new_df[new_df["fk_med_key"].notna()] + new_df.drop_duplicates(subset=["pk_meds_dispensed","fk_visit_id", "fk_med_key"], inplace=True) + print(f"Table ({table_name}) lost ({len(df) - len(new_df)}) entries") + df = new_df + elif table_name == "diagnoses_made": + new_df = df[df["pk_diagnoses_made"].notna()] + new_df = new_df[new_df["fk_visit_id"].notna()] + new_df = new_df[new_df["fk_diagnosis_key"].notna()] + new_df.drop_duplicates(subset=["pk_diagnoses_made", "fk_visit_id", "fk_diagnosis_key"], inplace=True) + print(f"Table ({table_name}) lost ({len(df) - len(new_df)}) entries") + df = new_df + elif table_name == "settings": + df['pk_settings_id'] = df.index + elif table_name == "patients": + df.drop(["face_photo"], axis=1, inplace=True) + query = "SELECT GetAs(face_photo, 'JPEG') FROM patients" + result = fmp.cursor.execute(query) + data = result.fetchall() + data = [item[0] for item in data] + df["face_photo"] = data + + + psql.write_table(df, table_name, if_exists="replace") + + pk_field, fk_fields = get_pk_and_fk_from_table(fmp, df, table_name) + all_data[table_name] = (pk_field, fk_fields) + + filename = "table_data.npy" + save_keys(filename, all_data) + + + filename = "table_data.npy" + all_data = load_dict(filename) + + for table_name in all_data.keys(): + if table_name not in fmp._join_tables: + pk_field, _ = all_data[table_name] + if pk_field is not None: + psql.execute_command(f"ALTER TABLE {table_name} ALTER COLUMN {pk_field} TYPE numeric USING {pk_field}::numeric;") + psql.execute_command(f"ALTER TABLE {table_name} ADD PRIMARY KEY ({pk_field});") + else: + if table_name == "meds_dispensed": + for pk_field in ["pk_meds_dispensed", "fk_visit_id"]: + psql.execute_command(f"ALTER TABLE {table_name} ALTER COLUMN {pk_field} TYPE numeric USING {pk_field}::numeric;") + psql.execute_command(f"ALTER TABLE {table_name} ADD PRIMARY KEY (pk_meds_dispensed, fk_visit_id, fk_med_key);") + elif table_name == "diagnoses_made": + for pk_field in ["pk_diagnoses_made", "fk_visit_id"]: + psql.execute_command(f"ALTER TABLE {table_name} ALTER COLUMN {pk_field} TYPE numeric USING {pk_field}::numeric;") + psql.execute_command(f"ALTER TABLE {table_name} ADD PRIMARY KEY (pk_diagnoses_made, fk_visit_id, fk_diagnosis_key);") + + + for table_name in all_data.keys(): + _, fk_fields = all_data[table_name] + + for fk_field, fk_table in fk_fields: + pk_fk_table = all_data[fk_table][0] + # ASSUMES WE ARE ALWAYS LINKING TO THE PK OF THE TABLE + constraint_name = f"fk_{fk_table}_id" + + psql.execute_command(f"ALTER TABLE {table_name} ALTER COLUMN {fk_field} TYPE numeric USING {fk_field}::numeric;") + + ''' + psql.execute_command( + f"ALTER TABLE {table_name} DROP CONSTRAINT IF EXISTS {constraint_name};" + ) + ''' + + try: + psql.execute_command( + f"ALTER TABLE {table_name} ADD CONSTRAINT {constraint_name} FOREIGN KEY ({fk_field}) REFERENCES {fk_table}({pk_fk_table});" + ) + except Exception as e: + print(f"FAILURE TO IMPLEMENT CONSTRAINT ON ({table_name}.{fk_field}) to ({fk_table}.{pk_fk_table})") + + +if __name__ == "__main__": + sys.exit(main(parse_args())) diff --git a/db/logs/20200425.log b/db/logs/20200425.log new file mode 100644 index 0000000..089dd76 --- /dev/null +++ b/db/logs/20200425.log @@ -0,0 +1,58 @@ + 0%| | 0/10 [00:00 pd.DataFrame: + # TODO: implement read_table functionality in PSQL + print("NEED TO IMPLEMENT THIS") + + def write_table( + self, data_frame: pd.DataFrame, table_name: str, if_exists: str = "fail" + ): + data_frame.to_sql( + con=self.__engine, name=table_name, if_exists=if_exists, index=False + ) + + # ########################### + + def open_connection(self): + return self.__engine.connect() + + def execute_command(self, command_string): + print(f"PSQL EXECUTING: [{command_string}]") + result = None + try: + with self.open_connection() as connection: + result = connection.execute(command_string) + except Exception as e: + print(f"Failure to execute: ['{command_string}']") + raise (e) # TODO: determine if crashing is appropriate + + return result diff --git a/db/query_fmp.py b/db/query_fmp.py new file mode 100644 index 0000000..f64ad75 --- /dev/null +++ b/db/query_fmp.py @@ -0,0 +1,58 @@ +import argparse +import os +import pickle +import sys +import pyodbc + +import pandas as pd +from tqdm import tqdm + +from fmp import FMP +from psql import PSQL +from utils import edit_distance +from PIL import Image +import io + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--dsn", help="dsn name") + parser.add_argument("-u", "--username", help="username for dsn") + parser.add_argument("-p", "--password", help="password for dsn") + return parser.parse_args() + + +def main(argv): + fmp = FMP(argv.dsn, argv.username, argv.password) + # query = "SELECT GetAs(face_photo, 'JPEG') FROM patients WHERE pk_patient_id=31883 OR pk_patient_id=1" + # result = fmp.cursor.execute(query) + # data = result.fetchall() + # data = [item[0] for item in data] + # print(data, len(data)) + # images = [Image.open(io.BytesIO(d)) for d in data] + # [image.show() for image in images] + + + # table_name = "diagnoses_made" + # print(table_name) + # res_1 = fmp.read_table(table_name) + # print(res_1) + + table_name = "globals" + print(table_name) + res_2 = fmp.read_table(table_name) + print(res_2.columns) + print(res_2) + # print(res_1 == res_2) + + # table_name = "clinics" + # print(table_name) + # res_3 = fmp.read_table(table_name) + # print(res_3) + + # print(type(fmp.execute_command("SELECT face_photo FROM patients WHERE pk_patient_id=31883")["face_photo"][0])) + # print(fmp.execute_command("SELECT face_photo FROM patients WHERE pk_patient_id=31883")["face_photo"][0]) + + +if __name__ == "__main__": + sys.exit(main(parse_args())) diff --git a/db/reset_postgres.py b/db/reset_postgres.py new file mode 100644 index 0000000..7ea58ac --- /dev/null +++ b/db/reset_postgres.py @@ -0,0 +1,8 @@ +from psql import PSQL + +psql = PSQL("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db") + +psql.execute_command("DROP SCHEMA public CASCADE") +psql.execute_command("CREATE SCHEMA public") +psql.execute_command("GRANT ALL ON SCHEMA public TO postgres") +psql.execute_command("GRANT ALL ON SCHEMA public TO public") diff --git a/db/utils.py b/db/utils.py new file mode 100644 index 0000000..ea9021d --- /dev/null +++ b/db/utils.py @@ -0,0 +1,22 @@ +def edit_distance(str1, str2): + return edit_distance_helper(str1, str2, len(str1) - 1, len(str2) - 1) + + +def edit_distance_helper(str1, str2, m, n): + if m == 0: + return n + + if n == 0: + return m + + if str1[m] == str2[n]: + return edit_distance_helper(str1, str2, m - 1, n - 1) + + return ( + min( + edit_distance_helper(str1, str2, m, n - 1), + edit_distance_helper(str1, str2, m - 1, n), + edit_distance_helper(str1, str2, m - 1, n - 1), + ) + + 1 + ) diff --git a/dev-docker-compose.yml b/dev-docker-compose.yml index 3fcc8f0..f90730d 100644 --- a/dev-docker-compose.yml +++ b/dev-docker-compose.yml @@ -24,7 +24,7 @@ services: ports: - "5432:5432" expose: - - "5422" + - "5432" environment: POSTGRES_DB: app_db POSTGRES_USER: postgres diff --git a/pi-docker-compose.yml b/pi-docker-compose.yml index ff7c11c..33dbdef 100644 --- a/pi-docker-compose.yml +++ b/pi-docker-compose.yml @@ -16,7 +16,7 @@ services: database: image: "postgres" # use latest official postgres version env_file: - - database.env # configure postgres + - ./database.env # configure postgres volumes: - "database-data:/var/lib/postgresql/data/" # persist data even if container shuts down ports: