SAVE: working w/ zain irl

This commit is contained in:
abpwrs committed 2020-03-29 14:35:39 -05:00
1 parent 2ecbff1def
commit 6762152b1e
27 files changed
+1886 -145

No files matched your search

-103
View File
@@ -1,103 +0,0 @@
import pyodbc
"""
File Maker Pro Connection Object
"""
class FMP(object):
__connection_string = "DSN=chifmp;UID=hjohnson;PWD=hjohnson"
def __init__(self):
self.fmp_table_names = [
"clinics",
"patients",
"visits",
"diagnoses",
"diagnoses_made",
"meds",
"meds_dispensed",
# "settings",
# "globals",
]
self.python_types_to_sql = {
"str": "VARCHAR",
"date": "TIMESTAMP",
"float64": "FLOAT",
"NoneType": "VARCHAR",
}
self.__connection = pyodbc.connect(FMP.__connection_string)
def fmp_table_names(self):
return self.fmp_table_names
@property
def connection(self):
return self.__connection
class CHIFMPModel(object):
def __init__(self):
self.chifmp_table_names = [
"patients",
"visits",
"clinics",
"diagnoses",
"diagnoses_made",
"meds",
"meds_dispensed",
# "globals",
# "settings",
]
def get_chifmp_table_names(self):
return self.chifmp_table_names
def _get_connection_string(self):
# import netrc
## Add secret information in the `~/.netrc file
# machine chihaiti.fmp
# account chihaiti #<- the name of the DSN to connetct to
# login fmp_database_name
# password fmp_database_password
# netrcparser = netrc.netrc()
# mylogin, myaccount, mypassword = netrcparser.authenticators("chihaiti.fmp")
connection_string = "DSN=chifmp;UID=hjohnson;PWD=hjohnson"
print("Starting Connection with {0}".format(connection_string))
# del mylogin, myaccount, mypassword
return connection_string
def get_connnection(self):
import pyodbc
return pyodbc.connect(self._get_connection_string())
class QueryElements(object):
"""
Class to hold components used to construct a query
updates should be done like the following:
template = "UPDATE patients SET first_name = ? WHERE pk_patient_id = ?"
values = ('My"Name"', 12342)
cursor.execute(template, values);
"""
def __init__(self, query_template, query_values):
self.query_template = query_template
self.query_values = query_values
def get_query_template(self):
return self.query_template
def get_query_values(self):
return self.query_values
def __str__(self):
return str(self.query_template) + " " + str(self.query_values)
-38
View File
@@ -1,38 +0,0 @@
import json
from datetime import datetime
import pandas as pd
import pyodbc
from fmp import FMP
db_json = dict()
fmp_model = FMP()
for tbl in fmp_model.fmp_table_names:
query = "select * from {tbl}".format(tbl=tbl)
df = pd.read_sql(query, fmp_model.connection)
print(tbl)
col_types = [
df.iloc[1, index].__class__.__name__ for index in range(len(df.columns))
]
for elem in zip(df.columns, col_types):
print(elem)
print()
db_json[tbl] = {}
col_types = [
df.iloc[1, index].__class__.__name__ for index in range(len(df.columns))
]
for elem in zip(df.columns, col_types):
db_json[tbl][elem[0]] = {}
db_json[tbl][elem[0]]["type"] = elem[1]
db_json[tbl][elem[0]]["pk"] = "pk" == elem[0][:2]
db_json[tbl][elem[0]]["fk"] = "fk" == elem[0][:2]
curr_time = datetime.now().strftime("%Y%m%d_%H%M")
with open("./models/{time}_model.json".format(time=curr_time), "w") as f:
json.dump(db_json, f, indent=4)
File renamed without changes.
File renamed without changes.
View File
File renamed without changes.
+19
View File
@@ -0,0 +1,19 @@
from abc import ABC, abstractmethod
class DB(ABC):
def __init__(self):
self._model = None
self.datetime_format = "%Y%m%d_%H%M"
@abstractmethod
def write_model(self):
pass
@abstractmethod
def parse_database(self) -> dict():
pass
@abstractmethod
def init_from_json(self, json_model: dict()):
pass
File renamed without changes.
+1
View File
@@ -0,0 +1 @@
psql --host=database --username=user --dbname=database
+9
View File
@@ -0,0 +1,9 @@
query = "SELECT tc.table_schema, tc.constraint_name, tc.table_name, kcu.column_name, ccu.table_schema AS foreign_table_schema, ccu.table_name AS foreign_table_name, ccu.column_name AS foreign_column_name FROM information_schema.table_constraints AS tc JOIN information_schema.key_column_usage AS kcu ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema JOIN information_schema.constraint_column_usage AS ccu ON ccu.constraint_name = tc.constraint_name AND ccu.table_schema = tc.table_schema WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='patients';"
from fmp import FMP
import pandas as pd
fmp_model = FMP()
df = pd.read_sql(query, fmp_model.connection)
print(df)
+17
View File
@@ -0,0 +1,17 @@
SELECT
tc.table_schema,
tc.constraint_name,
tc.table_name,
kcu.column_name,
ccu.table_schema AS foreign_table_schema,
ccu.table_name AS foreign_table_name,
ccu.column_name AS foreign_column_name
FROM
information_schema.table_constraints AS tc
JOIN information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
JOIN information_schema.constraint_column_usage AS ccu
ON ccu.constraint_name = tc.constraint_name
AND ccu.table_schema = tc.table_schema
WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='patients';
+98
View File
@@ -0,0 +1,98 @@
import json
from datetime import datetime
import pandas as pd
import pyodbc
from tqdm import tqdm
from db import DB
"""
File Maker Pro Connection Object
Desc:
This file contains the primary logic for interacting with the file maker pro database
"""
class FMP(DB):
def __init__(self, dsn=None, username=None, password=None):
DB.__init__(self)
if dsn is None:
dsn = input("Enter the FMP DSN: ")
if username is None:
username = input("Enter your FMP username: ")
if password is None:
password = input("Enter your FMP password: ")
self.__dsn = dsn
self.__username = username
self.__password = password
self.__connection_string = "DSN={0};UID={1};PWD={2}".format(
self.__dsn, self.__username, self.__password
)
self.__fmp_table_names = [
"clinics",
"patients",
"visits",
"diagnoses",
"diagnoses_made",
"meds",
"meds_dispensed",
# "settings",
# "globals",
]
self.__python_types_to_sql = {
"str": "VARCHAR",
"date": "TIMESTAMP",
"float64": "FLOAT",
"NoneType": "VARCHAR",
}
self.__pyodbc_connection = pyodbc.connect(self.__connection_string)
@property
def python_types_to_sql(self):
return self.__python_types_to_sql
@property
def fmp_table_names(self):
return self.__fmp_table_names
@property
def connection(self):
return self.__pyodbc_connection
# ABSTRACT BASE CLASS MEHTODS
def init_from_json(model_object):
self._model = model_object
def parse_database(self) -> dict():
self._model = dict()
for tbl in tqdm(self.fmp_table_names):
query = "select * from {tbl}".format(tbl=tbl)
df = pd.read_sql(query, self.connection)
self._model[tbl] = {}
col_types = [
df.iloc[1, index].__class__.__name__ for index in range(len(df.columns))
]
for elem in zip(df.columns, col_types):
self._model[tbl][elem[0]] = {}
self._model[tbl][elem[0]]["type"] = elem[1]
self._model[tbl][elem[0]]["pk"] = "pk" == elem[0][:2]
self._model[tbl][elem[0]]["fk"] = "fk" == elem[0][:2]
return self._model
def write_model(self, model_object=None):
curr_time = datetime.now().strftime(self.datetime_format)
with open("./models/{time}_model.json".format(time=curr_time), "w") as f:
if model_object is not None:
json.dump(model_object, f, indent=4)
else:
json.dump(self._model, f, indent=4)
File renamed without changes.
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Author: "hertz"
# Date: 20200310
# args:
# 1 --
# 2 --
#if [[ $# -ne 1 ]]; then
# echo "Usage: $0 <param-1> <param-2> ..."
# exit
#fi
# script:
@@ -238,7 +238,7 @@ create table chi_20200305_2357.visits (
create table chi_20200305_2357.diagnoses (
pk_diagnosis_id FLOAT primary key,
diagnosis_name VARCHAR ,
pk_diagnosis_key VARCHAR primary key,
pk_diagnosis_key VARCHAR ,
creation_date TIMESTAMP ,
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
@@ -262,7 +262,7 @@ create table chi_20200305_2357.meds (
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
modified_acct VARCHAR ,
pk_med_key VARCHAR primary key,
pk_med_key VARCHAR ,
med_type VARCHAR
);
+280
View File
@@ -0,0 +1,280 @@
create schema chi_20200309_1938;
create table chi_20200309_1938.clinics (
pk_clinic_id FLOAT primary key,
clinic_name VARCHAR ,
creation_date TIMESTAMP ,
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
modified_acct VARCHAR
);
create table chi_20200309_1938.patients (
pk_patient_id FLOAT primary key,
creation_date TIMESTAMP ,
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
modified_acct VARCHAR ,
last_name VARCHAR ,
first_name VARCHAR ,
gender VARCHAR ,
face_photo VARCHAR ,
birthdate VARCHAR ,
phone VARCHAR ,
town VARCHAR ,
unique_name VARCHAR ,
unique_ID_name VARCHAR ,
nickname VARCHAR ,
needs_review VARCHAR ,
notes_modification_status VARCHAR ,
fk_geonameid VARCHAR
);
create table chi_20200309_1938.visits (
pk_visit_id FLOAT primary key,
creation_date TIMESTAMP ,
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
modified_acct VARCHAR ,
fk_patient_id FLOAT ,
fk_clinic_id FLOAT ,
visit_date TIMESTAMP ,
visit_age FLOAT ,
ob_pregnant VARCHAR ,
ob_last_menstrual_period VARCHAR ,
bp_systolic_blood_pressure FLOAT ,
bp_diastolic_blood_pressure FLOAT ,
bp_mean_arterial_pressure FLOAT ,
temp_in_fahrenheit FLOAT ,
pulse_rate FLOAT ,
weight_in_lbs FLOAT ,
height_in_inches FLOAT ,
glasses_given VARCHAR ,
sunglasses_given VARCHAR ,
dx_surgical_candidate VARCHAR ,
surgical_comments VARCHAR ,
visit_age_unit VARCHAR ,
chief_complaint VARCHAR ,
history_of_present_illness VARCHAR ,
visit_age_years FLOAT ,
exam_comments VARCHAR ,
tests_comments VARCHAR ,
tests_blood_sugar_result FLOAT ,
tests_pregnancy_result VARCHAR ,
tests_urine_glucose_result VARCHAR ,
tests_urine_protein_result VARCHAR ,
tests_urine_leukocytes_result VARCHAR ,
tests_urine_nitrites_result VARCHAR ,
ob_gravida FLOAT ,
ob_para FLOAT ,
ob_abortus FLOAT ,
ob_history VARCHAR ,
ob_wants_planning VARCHAR ,
albendazole_given VARCHAR ,
first_visit VARCHAR ,
known_med_allergies VARCHAR ,
meds_currently_taken VARCHAR ,
tests_pregnancy_given VARCHAR ,
tests_blood_sugar_given VARCHAR ,
tests_eye_exam_given VARCHAR ,
tests_urine_analysis_given VARCHAR ,
dx_asthma VARCHAR ,
dx_allergies VARCHAR ,
dx_anemia VARCHAR ,
dx_malnutrition VARCHAR ,
dx_diabetes VARCHAR ,
dx_pterygium VARCHAR ,
dx_cataract VARCHAR ,
dx_dry_eye VARCHAR ,
dx_conjunctivitis VARCHAR ,
dx_fever VARCHAR ,
dx_possible_malaria VARCHAR ,
dx_filariasis VARCHAR ,
dx_gastritis VARCHAR ,
dx_gerd VARCHAR ,
dx_gastroenteritis VARCHAR ,
dx_dysentery VARCHAR ,
dx_possible_cholera VARCHAR ,
dx_headache VARCHAR ,
dx_hernia VARCHAR ,
dx_hernia_type VARCHAR ,
dx_hypertension VARCHAR ,
dx_stroke VARCHAR ,
dx_angina VARCHAR ,
dx_chf VARCHAR ,
dx_musculoskeletal_pain VARCHAR ,
dx_otitis VARCHAR ,
dx_otitis_ear VARCHAR ,
dx_pneumonia VARCHAR ,
dx_sinusitis VARCHAR ,
dx_uri VARCHAR ,
dx_pregnant VARCHAR ,
dx_pregnant_weeks FLOAT ,
dx_atopic_derm VARCHAR ,
dx_fungal VARCHAR ,
dx_scabies VARCHAR ,
dx_impetigo VARCHAR ,
dx_tinea_capitus VARCHAR ,
dx_possible_std VARCHAR ,
dx_possible_tb VARCHAR ,
dx_vaginitis_candidal VARCHAR ,
dx_vaginitis_trich VARCHAR ,
dx_vaginitis_bv VARCHAR ,
dx_other VARCHAR ,
dx_other_text VARCHAR ,
tx_multivitamin VARCHAR ,
tx_pain_acetaminophen VARCHAR ,
tx_gi_omeprazole VARCHAR ,
tx_gi_pepto_bismol VARCHAR ,
tx_gi_cimetidine VARCHAR ,
tx_gi_tums VARCHAR ,
tx_gi_ors VARCHAR ,
tx_infections_albendazole VARCHAR ,
tx_infections_amoxicillin VARCHAR ,
tx_infections_azithromycin VARCHAR ,
tx_infections_bactrim VARCHAR ,
tx_infections_cipro VARCHAR ,
tx_infections_doxycycline VARCHAR ,
tx_infections_flagyl VARCHAR ,
tx_infections_fluconazole VARCHAR ,
tx_infections_griseofulvin VARCHAR ,
tx_infections_ivermectin VARCHAR ,
tx_infections_keflex VARCHAR ,
tx_infections_rocephin VARCHAR ,
tx_cardiac_amlodipine VARCHAR ,
tx_cardiac_aspirin VARCHAR ,
tx_cardiac_atenolol VARCHAR ,
tx_cardiac_hctz VARCHAR ,
tx_cardiac_lasix VARCHAR ,
tx_cardiac_enalapril VARCHAR ,
tx_topical_clotrimazole VARCHAR ,
tx_topical_hydrocortisone VARCHAR ,
tx_topical_silvadine VARCHAR ,
tx_topical_permethirine VARCHAR ,
tx_im_dexamethasone VARCHAR ,
tx_im_lasix VARCHAR ,
tx_misc_albuterol VARCHAR ,
tx_misc_benadryl VARCHAR ,
tx_misc_ferrous_sulfate VARCHAR ,
tx_misc_metformin VARCHAR ,
tx_misc_prednisolone VARCHAR ,
tx_misc_other VARCHAR ,
tx_misc_other_text VARCHAR ,
tx_comments VARCHAR ,
dx_uti VARCHAR ,
tx_multivitamin_varieties VARCHAR ,
tx_pain_acetaminophen_varieties VARCHAR ,
tx_infections_amoxicillin_varieties VARCHAR ,
tx_pain_ibuprofen VARCHAR ,
dx_comments VARCHAR ,
tx_infections_azithromycin_varieties VARCHAR ,
tx_infections_bactrim_varieties VARCHAR ,
tx_infections_keflex_varieties VARCHAR ,
tx_infections_rocephin_varieties VARCHAR ,
dx_vaginitis_general VARCHAR ,
dx_needs_to_see_dentist VARCHAR ,
dx_needs_to_see_eye_doctor VARCHAR ,
dx_healthy VARCHAR ,
provider VARCHAR ,
tx_saw_dentist VARCHAR ,
tx_saw_eye_doctor VARCHAR ,
tx_outside_referral VARCHAR ,
tx_outside_referral_text VARCHAR ,
tx_gi_zinc VARCHAR ,
tx_cardiac_methyldopa VARCHAR ,
tx_infections_augmentin VARCHAR ,
tx_infections_chloroquine VARCHAR ,
tx_infections_griseofuvin_varieties VARCHAR ,
tx_infections_nitrofurantoin VARCHAR ,
tx_topical_double_antibiotic VARCHAR ,
tx_misc_fluoxetine VARCHAR ,
tx_misc_ocp VARCHAR ,
tx_misc_phenytoin VARCHAR ,
dx_possible_parasites VARCHAR ,
tx_misc_iud VARCHAR ,
tests_id_test_given VARCHAR ,
tests_id_chlamydia_result VARCHAR ,
tests_id_hiv_result VARCHAR ,
tests_id_syphilis_result VARCHAR ,
tests_id_gonorrhea_result VARCHAR ,
tests_eye_exam_right_sphere VARCHAR ,
tests_eye_exam_right_cylinder VARCHAR ,
tests_eye_exam_right_axis VARCHAR ,
tests_eye_exam_right_bifocal VARCHAR ,
tests_eye_exam_left_sphere VARCHAR ,
tests_eye_exam_left_cylinder VARCHAR ,
tests_eye_exam_left_axis VARCHAR ,
tests_eye_exam_left_bifocal VARCHAR ,
tx_pain_ibuprofen_varieties VARCHAR ,
tx_misc_benadryl_varieties VARCHAR ,
tests_id_tb_result VARCHAR ,
tests_id_confirm_hiv_cd4 VARCHAR ,
tests_id_confirm_hiv_result VARCHAR ,
tests_id_confirm_hiv_arvt VARCHAR ,
tests_id_confirm_hiv_enrolled VARCHAR ,
tests_id_confirm_hiv_problems VARCHAR ,
tests_id_confirm_tb_result VARCHAR ,
tests_id_confirm_tb_comments VARCHAR ,
tests_id_confirm_hiv_enrolled_at VARCHAR ,
dx_depression VARCHAR ,
clinic_trip VARCHAR ,
tx_chw_referral VARCHAR ,
tx_misc_depo VARCHAR ,
tx_misc_paroxetine VARCHAR ,
tx_misc_keppra VARCHAR ,
dx_totalcount FLOAT ,
tx_totalcount FLOAT ,
tx_given_glasses VARCHAR ,
tx_misc_labetolol VARCHAR ,
tx_misc_carbamazepine VARCHAR ,
tx_misc_terazosin VARCHAR ,
tx_gi_ranitidine VARCHAR ,
tx_cardiac_lisinopril VARCHAR ,
tx_topical_triple_antibiotic VARCHAR ,
tx_misc_prednisone VARCHAR ,
clinic VARCHAR ,
notes_modification_status VARCHAR
);
create table chi_20200309_1938.diagnoses (
pk_diagnosis_id FLOAT primary key,
diagnosis_name VARCHAR ,
pk_diagnosis_key VARCHAR ,
creation_date TIMESTAMP ,
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
modified_acct VARCHAR
);
create table chi_20200309_1938.diagnoses_made (
pk_diagnoses_made FLOAT primary key,
fk_visit_id FLOAT ,
fk_diagnosis_key VARCHAR ,
creation_date TIMESTAMP ,
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
modified_acct VARCHAR
);
create table chi_20200309_1938.meds (
pk_med_id FLOAT primary key,
med_name VARCHAR ,
creation_date TIMESTAMP ,
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
modified_acct VARCHAR ,
pk_med_key VARCHAR ,
med_type VARCHAR
);
create table chi_20200309_1938.meds_dispensed (
pk_meds_dispensed FLOAT primary key,
fk_visit_id FLOAT ,
fk_med_key VARCHAR ,
creation_date TIMESTAMP ,
creation_acct VARCHAR ,
modified_date TIMESTAMP ,
modified_acct VARCHAR ,
dispensed_total FLOAT ,
med_dosage VARCHAR
);
+30
View File
@@ -0,0 +1,30 @@
import argparse
import sys
from fmp import FMP
"""
Generate JSON from file maker pro
DESC:
This file will take in a data source name (dsn), username, and password, and generate to the
best of it's abilities a json file that represents the structure of the database
"""
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--dsn", help="dsn name")
parser.add_argument("-u", "--username", help="username for dsn")
parser.add_argument("-p", "--password", help="password for dsn")
return parser.parse_args()
def main(argv):
fmp = FMP(argv.dsn, argv.username, argv.password)
model = fmp.parse_database()
fmp.write_model(model)
if __name__ == "__main__":
sys.exit(main(parse_args()))
File renamed without changes.
File renamed without changes.
File renamed without changes.
File diff suppressed because it is too large. Load diff
+79
View File
@@ -0,0 +1,79 @@
import argparse
import json
import os
import sys
from datetime import datetime
from fmp import FMP
db_json = dict()
fmp_model = FMP()
# 1st (sql file)
# 2nd (write to sql connection)
def write_command(string, out_file):
with open(out_file, "w") as f:
f.write(string)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model", required=True, help="JSON model file to seed")
args = parser.parse_args()
print(args)
if not os.path.isfile(os.path.abspath(args.model)):
raise ValueError("File: {0} doesn't exist".format(args.model))
return args
def main(argv):
model_file = argv.model
with open(model_file, "r") as f:
data_model = json.load(f)
date_time = "_".join(os.path.split(os.path.abspath(model_file))[-1].split("_")[:2])
output_file = "./init/{date_time}_init_db.sql".format(date_time=date_time)
schema = "chi_{date_time}".format(date_time=date_time)
command = "create schema {schema};\n".format(schema=schema)
for table_name, attribute_dict in data_model.items():
command += "create table {schema}.{table_name} (\n".format(
schema=schema, table_name=table_name
)
features = list(attribute_dict.keys())
last_elem = features[-1]
# HACK: mark the first feautre name with 'pk' in
# the name as the primary key even if others exist.
pk_found = False
for feature in features:
extra_param_string = ""
if attribute_dict[feature]["pk"] and not pk_found:
extra_param_string += "primary key"
pk_found = True
# elif attribute_dict[feature]["fk"]:
# extra_param_string += "foreign key"
command += "\t{attr_name} {attr_type} {extra_params}".format(
attr_name=feature,
attr_type=fmp_model.python_types_to_sql[
attribute_dict[feature]["type"]
],
extra_params=extra_param_string,
)
if feature != last_elem:
command += ",\n"
else:
command += "\n"
command += ");\n\n"
write_command(command, output_file)
if __name__ == "__main__":
sys.exit(main(parse_args()))
@@ -32,7 +32,7 @@ def main(argv):
with open(model_file, "r") as f:
data_model = json.load(f)
date_time = '_'.join(os.path.split(os.path.abspath(model_file))[-1].split('_')[:2])
date_time = "_".join(os.path.split(os.path.abspath(model_file))[-1].split("_")[:2])
schema = "chi_{date_time}".format(date_time=date_time)
command = "create schema {schema};\n".format(schema=schema)
for table_name, attribute_dict in data_model.items():
View File
Whitespace-only changes.
+7
View File
@@ -0,0 +1,7 @@
# import some python ORM for SQl
from db import DB
class PSQL(DB):
def __init__(self,):
pass
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Author: "hertz"
# Date: 20200310
# args:
# 1 --
# 2 --
#if [[ $# -ne 1 ]]; then
# echo "Usage: $0 <param-1> <param-2> ..."
# exit
#fi
# script:
+2
View File
@@ -0,0 +1,2 @@
*
!.gitignore
+1 -1
View File
@@ -13,7 +13,7 @@ services:
database:
image: "postgres" # use latest official postgres version
env_file:
- database.env # configure postgres
- ./db/database.env # configure postgres
volumes:
- database-data:/var/lib/postgresql/data/ # persist data even if container shuts down
ports: