diff --git a/db/.old/json_from_fmp.py b/db/.old/json_from_fmp.py index f2729ad..6061291 100644 --- a/db/.old/json_from_fmp.py +++ b/db/.old/json_from_fmp.py @@ -3,6 +3,7 @@ import sys from fmp import FMP + """ Generate JSON from file maker pro diff --git a/db/.old/seed_postgres.py b/db/.old/seed_postgres.py index 9250566..d605e14 100644 --- a/db/.old/seed_postgres.py +++ b/db/.old/seed_postgres.py @@ -1,11 +1,10 @@ import argparse import os -import pandas as pd import sys -from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine - +import pandas as pd from fmp import FMP +from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine engine = create_engine("postgres+psycopg2://user:password@0.0.0.0:5432/database") diff --git a/db/clean-python.bash b/db/clean-python.bash new file mode 100755 index 0000000..5e65ecd --- /dev/null +++ b/db/clean-python.bash @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Author: "abpwrs" +# Date: 20200409 + +# script: +isort -rc . +black . + diff --git a/db/db.py b/db/db.py index 263fef0..d9d2a8c 100644 --- a/db/db.py +++ b/db/db.py @@ -1,11 +1,23 @@ from abc import ABC, abstractmethod +import pandas as pd +from utils import edit_distance + + +""" +DESC: + Abstract Base Class (ABC) for a Database Object + Extracts shared logic from the FMP and PSQL classes +""" + class DB(ABC): def __init__(self): self._model = None self.datetime_format = "%Y%m%d_%H%M" + self._table_names = [] + # TODO: Delete these methods once we remove our custom model code @abstractmethod def write_model(self): pass @@ -14,5 +26,57 @@ class DB(ABC): def parse_database(self) -> dict(): pass + # ABSTRACT METHDOS TO BE IMPLEMENTED BY CHILD CLASSES + # ################################################### + + @abstractmethod + def read_table(self, table_name) -> pd.DataFrame: + pass + + @abstractmethod + def write_table(self, data: pd.DataFrame, table_name): + pass + + # ################################################### + + @property + def table_names(self) -> list: + return self._table_names + + def get_tablename_from_fieldname(self, field_name: str): + try: + assert field_name[:3] == "pk_" or field_name[:3] == "fk_" + except AssertionError as e: + print( + "Can only get meaningful table names from field names on primary or foreign keys" + ) + raise (e) + + # remove "pk_" or "fk_" --> we only enter this function if this has already matched + extracted_field_to_match = field_name[3:] + + # not all fields that start with "pk_" or "fk_" end in "_id" + if field_name[-3:] == "_id": + extracted_field_to_match = extracted_field_to_match[:-3] + else: + print(f"consider fixing fieldname: {field_name}") + + # function that returns the edit distance of the table to the extracted field name + matcher = lambda table_name: edit_distance(table_name, extracted_field_to_match) + + try: + # map lambda over list of table `names & extract min edit distance + index_of_match = int( + np.argmin(np.array(list(map(matcher, self.table_names))), axis=0) + ) + except TypeError as e: + print("FAILURE: two tables have equally close edit_distance") + # TODO: potentially add a backup lookup table here + # for now we will re-raise the error and crash the program + raise (e) + + return self.table_names[index_of_match] + + # TODO: delete this once we adjust to the new dual connection functionality def init_from_json(self, json_model: dict()): self._model = json_model diff --git a/db/execute-sql-in-container.py b/db/execute-sql-in-container.py index ffaf0ea..760c2be 100755 --- a/db/execute-sql-in-container.py +++ b/db/execute-sql-in-container.py @@ -13,9 +13,10 @@ import argparse -import sys import pickle +import sys +from psql import PSQL from sqlalchemy import create_engine engine = create_engine("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db") @@ -28,8 +29,10 @@ def parse_args(): def main(argv): - with engine.connect() as con: - result = con.execute(argv.command) + # TODO: extract this from database.env (ideally but it in the PSQL class) + psql = PSQL("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db") + result = psql.execute_command(argv.command) + print(result) if __name__ == "__main__": diff --git a/db/fmp.py b/db/fmp.py index e4a7b7d..02fb62d 100644 --- a/db/fmp.py +++ b/db/fmp.py @@ -4,11 +4,11 @@ from datetime import datetime import numpy as np import pandas as pd import pyodbc -from tqdm import tqdm - from db import DB +from tqdm import tqdm from utils import edit_distance + """ File Maker Pro Connection Object @@ -35,7 +35,8 @@ class FMP(DB): self.__dsn, self.__username, self.__password ) - self.__fmp_table_names = [ + # TODO: consider extracting this list to the DB class + self._table_names = [ "clinics", "patients", "visits", @@ -45,8 +46,8 @@ class FMP(DB): "meds_dispensed", "geonames", "settings" - # "settings", # "globals", + # TODO: add the join tables that are still in the FMP model, but weren't used last year ] self.__python_types_to_sql = { @@ -60,6 +61,20 @@ class FMP(DB): if connect: self.__connect() + # METHODS FROM DB SUPER CLASS + # ########################### + + def read_table(self, table_name) -> pd.DataFrame: + query = "select * from {table}".format(table=table_name) + df = pd.read_sql(query, self.connection) + return df + + def write_table(self, data, table_name): + # TODO: implement write_table in FMP class + print("IMPLEMENT WRITE_TABLE IN FMP") + + # ########################### + def __connect(self): if self.__pyodbc_connection is None: self.__pyodbc_connection = pyodbc.connect(self.__connection_string) @@ -68,10 +83,6 @@ class FMP(DB): def python_types_to_sql(self): return self.__python_types_to_sql - @property - def fmp_table_names(self) -> list: - return self.__fmp_table_names - @property def connection(self): if self.__pyodbc_connection is None: @@ -82,76 +93,50 @@ class FMP(DB): def parse_database(self) -> dict(): self._model = dict() - for tbl in tqdm(self.fmp_table_names): - query = "select * from {tbl}".format(tbl=tbl) - df = pd.read_sql(query, self.connection) + for table_name in tqdm(self.table_names): + df = self.read_table(table_name) - self._model[tbl] = {} + self._model[table_name] = {} col_types = [ df.iloc[0, index].__class__.__name__ for index in range(len(df.columns)) ] - smallest_pk_edit_distance = 123456789 + smallest_pk_edit_distance = float("inf") previous_pk_field = None for field_name, field_type in zip(df.columns, col_types): - self._model[tbl][field_name] = {} - self._model[tbl][field_name]["type"] = field_type + self._model[table_name][field_name] = {} + self._model[table_name][field_name]["type"] = field_type # extract primary keys if field_name[:3] == "pk_": # possible primary key - distance = edit_distance(field_name[3:-3], tbl) + distance = edit_distance(field_name[3:-3], table_name) if distance < smallest_pk_edit_distance: # closest primary key so far if previous_pk_field is not None: # remove previously assumed pk - self._model[tbl][previous_pk_field]["pk"] = False + self._model[table_name][previous_pk_field]["pk"] = False # set current pk - self._model[tbl][field_name]["pk"] = True + self._model[table_name][field_name]["pk"] = True previous_pk_field = field_name smallest_pk_edit_distance = distance else: - self._model[tbl][field_name]["pk"] = False + self._model[table_name][field_name]["pk"] = False else: - self._model[tbl][field_name]["pk"] = False + self._model[table_name][field_name]["pk"] = False # extract foreign keys if field_name[:3] == "fk_": - self._model[tbl][field_name]["fk"] = True - self._model[tbl][field_name][ + self._model[table_name][field_name]["fk"] = True + self._model[table_name][field_name][ "fk_table" ] = self.get_tablename_from_fieldname(field_name) else: - self._model[tbl][field_name]["fk"] = False + self._model[table_name][field_name]["fk"] = False return self._model - def get_tablename_from_fieldname(self, field_name: str): - # remove "pk_" or "fk_" --> we only enter this function if this has already matched - extracted_field_to_match = field_name[3:] - # not all fields that start with "pk_" or "fk_" end in "_id" - if field_name[-3:] == "_id": - extracted_field_to_match = extracted_field_to_match[:-3] - else: - print(f"consider fixing fieldname: {field_name}") - - # function that returns the edit distance of the table to the extracted field name - matcher = lambda table_name: edit_distance(table_name, extracted_field_to_match) - - try: - # map lambda over list of table `names & extract min edit distance - index_of_match = int( - np.argmin(np.array(list(map(matcher, self.fmp_table_names))), axis=0) - ) - except TypeError as e: - print("FAILURE: to tables have equally close edit_distance") - # TODO: potentially add a backup lookup table here - # for now we will re-raise the error and crash the program - raise (e) - - return self.__fmp_table_names[index_of_match] - def write_model(self, model_object: dict = None): curr_time = datetime.now().strftime(self.datetime_format) with open("./models/{time}_model.json".format(time=curr_time), "w") as f: diff --git a/db/fmp_to_postgres.py b/db/fmp_to_postgres.py index 82cac88..42a5d23 100644 --- a/db/fmp_to_postgres.py +++ b/db/fmp_to_postgres.py @@ -1,16 +1,12 @@ import argparse import os -import pandas as pd import sys -from tqdm import tqdm - -from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine +import pandas as pd from fmp import FMP +from tqdm import tqdm from utils import edit_distance -engine = create_engine("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db") - def parse_args(): parser = argparse.ArgumentParser() @@ -21,9 +17,9 @@ def parse_args(): def get_pk_and_fk_from_table(fmp: FMP, df: pd.DataFrame, table_name: str): - pk = None + pk = None fks = [] - smallest_pk_edit_distance = 1234567 + smallest_pk_edit_distance = float("inf") for field_name in df.columns: # extract primary keys if field_name[:3] == "pk_": @@ -42,30 +38,31 @@ def get_pk_and_fk_from_table(fmp: FMP, df: pd.DataFrame, table_name: str): def main(argv): fmp = FMP(argv.dsn, argv.username, argv.password) + # TODO: extract username, password, and database name from the database.env file + # TODO: encrypt the database.env file using git-secret + psql = PSQL("postgres+psycopg2://postgres:HaitiCHI@0.0.0.0:5432/app_db") all_data = dict() for table_name in tqdm(fmp.fmp_table_names): - query = "select * from {table_name}".format(table_name=table_name) - df = pd.read_sql(query, fmp.connection) - df.to_sql(con=engine, name=table_name, if_exists="replace", index=False) + df = fmp.read_table(table_name) + psql.write_table(df, table_name, if_exists="replace") pk_field, fk_fields = get_pk_and_fk_from_table(fmp, df, table_name) all_data[table_name] = (pk_field, fk_fields) for table_name in all_data.keys(): pk_field, fk_fields = all_data[table_name] + psql.execute_command(f"ALTER TABLE {table_name} ADD PRIMARY KEY ({pk_field});") + + # TODO: implement foreign key constraints + # for fk_field, fk_table in fk_fields: + # pk_fk_table = all_data[fk_table][0] + # # POTENTIALLY SUS: ASSUMES WE ARE ALWAYS LINKING TO THE PK OF THE TABLE + # constraint_name = f"fk_{fk_table}_id" + # psql.execute_command( + # f"ALTER TABLE {table_name} ADD CONSTRAINT {constraint_name} FOREIGN KEY ({fk_field}) REFERENCES {fk_table}({pk_fk_table});" + # ) - with engine.connect() as con: - con.execute( - f"ALTER TABLE {table_name} ADD PRIMARY KEY ({pk_field});" - ) - # for fk_field, fk_table in fk_fields: - # pk_fk_table = all_data[fk_table][0] - # # POTENTIALLY SUS: ASSUMES WE ARE ALWAYS LINKING TO THE PK OF THE TABLE - # constraint_name = f"fk_{fk_table}_id" - # con.execute( - # f"ALTER TABLE {table_name} ADD CONSTRAINT {constraint_name} FOREIGN KEY ({fk_field}) REFERENCES {fk_table}({pk_fk_table});" - # ) if __name__ == "__main__": sys.exit(main(parse_args())) diff --git a/db/psql.py b/db/psql.py index d66341f..bdb84a0 100644 --- a/db/psql.py +++ b/db/psql.py @@ -1,10 +1,39 @@ # import some python ORM for SQl -import sqlalchemy - +import pandas as pd from db import DB +from sqlalchemy import create_engine class PSQL(DB): - def __init__(self, postgres_db_name): + def __init__(self, db_uri): + self.__engine = create_engine(db_uri) - pass + # METHODS FROM DB SUPER CLASS + # ########################### + + def read_table(self, table_name) -> pd.DataFrame: + # TODO: implement read_table functionality in PSQL + print("NEED TO IMPLEMENT THIS") + + def write_table( + self, data_frame: pd.DataFrame, table_name: str, if_exists: str = "fail" + ): + data_frame.to_sql( + con=self.__engine, name=table_name, if_exists=if_exists, index=False + ) + + # ########################### + + def open_connection(self): + return self.__engine.connect() + + def execute_command(self, command_string): + result = None + try: + with self.open_connection() as connection: + result = connection.execute(command_string) + except Exception as e: + print(f"Failure to execute: ['{command_string}']") + raise (e) # TODO: determine if crashing is appropriate + + return result