ENH: adding fmp->json->postgres starter code

convert/.gitignore

@@ -0,0 +1 @@
+**/__pycache__/**

convert/fmp.py

@@ -0,0 +1,103 @@
+import pyodbc
+
+
+"""
+File Maker Pro Connection Object
+"""
+
+
+class FMP(object):
+    __connection_string = "DSN=chifmp;UID=hjohnson;PWD=hjohnson"
+
+    def __init__(self):
+        self.fmp_table_names = [
+            "clinics",
+            "patients",
+            "visits",
+            "diagnoses",
+            "diagnoses_made",
+            "meds",
+            "meds_dispensed",
+            # "settings",
+            # "globals",
+        ]
+
+        self.python_types_to_sql = {
+            "str": "VARCHAR",
+            "date": "TIMESTAMP",
+            "float64": "FLOAT",
+            "NoneType": "VARCHAR",
+        }
+
+        self.__connection = pyodbc.connect(FMP.__connection_string)
+
+    def fmp_table_names(self):
+        return self.fmp_table_names
+
+    @property
+    def connection(self):
+        return self.__connection
+
+
+class CHIFMPModel(object):
+    def __init__(self):
+        self.chifmp_table_names = [
+            "patients",
+            "visits",
+            "clinics",
+            "diagnoses",
+            "diagnoses_made",
+            "meds",
+            "meds_dispensed",
+            # "globals",
+            # "settings",
+        ]
+
+    def get_chifmp_table_names(self):
+        return self.chifmp_table_names
+
+    def _get_connection_string(self):
+        # import netrc
+
+        ## Add secret information in the `~/.netrc file
+        # machine chihaiti.fmp
+        #    account  chihaiti   #<- the name of the DSN to connetct to
+        #    login    fmp_database_name
+        #    password fmp_database_password
+        # netrcparser = netrc.netrc()
+        # mylogin, myaccount, mypassword = netrcparser.authenticators("chihaiti.fmp")
+        connection_string = "DSN=chifmp;UID=hjohnson;PWD=hjohnson"
+        print("Starting Connection with {0}".format(connection_string))
+        # del mylogin, myaccount, mypassword
+        return connection_string
+
+    def get_connnection(self):
+        import pyodbc
+
+        return pyodbc.connect(self._get_connection_string())
+
+
+class QueryElements(object):
+    """
+    Class to hold components used to construct a query
+
+    updates should be done like the following:
+
+    template = "UPDATE patients SET first_name = ? WHERE pk_patient_id = ?"
+    values = ('My"Name"', 12342)
+    cursor.execute(template, values);
+
+    """
+
+    def __init__(self, query_template, query_values):
+        self.query_template = query_template
+        self.query_values = query_values
+
+    def get_query_template(self):
+        return self.query_template
+
+    def get_query_values(self):
+        return self.query_values
+
+    def __str__(self):
+        return str(self.query_template) + " " + str(self.query_values)

convert/fmp_odbc_to_postgres.py

@@ -1,7 +0,0 @@
-import pyodbc 
-
-main():
-    print('I started')
-
-if __name__ == "main":
-    main()

convert/fmp_to_postgres.py

@@ -0,0 +1,9 @@
+import pyodbc
+
+
+def main():
+    print("I started")
+
+
+if __name__ == "main":
+    main()

convert/generate_json_from_fmp.py

@@ -0,0 +1,38 @@
+import json
+from datetime import datetime
+
+import pandas as pd
+import pyodbc
+
+from fmp import FMP
+
+db_json = dict()
+fmp_model = FMP()
+
+for tbl in fmp_model.fmp_table_names:
+    query = "select * from {tbl}".format(tbl=tbl)
+    df = pd.read_sql(query, fmp_model.connection)
+
+    print(tbl)
+    col_types = [
+        df.iloc[1, index].__class__.__name__ for index in range(len(df.columns))
+    ]
+    for elem in zip(df.columns, col_types):
+        print(elem)
+    print()
+
+    db_json[tbl] = {}
+    col_types = [
+        df.iloc[1, index].__class__.__name__ for index in range(len(df.columns))
+    ]
+    for elem in zip(df.columns, col_types):
+        db_json[tbl][elem[0]] = {}
+        db_json[tbl][elem[0]]["type"] = elem[1]
+        db_json[tbl][elem[0]]["pk"] = "pk" == elem[0][:2]
+        db_json[tbl][elem[0]]["fk"] = "fk" == elem[0][:2]
+
+
+curr_time = datetime.now().strftime("%Y%m%d_%H%M")
+
+with open("./models/{time}_model.json".format(time=curr_time), "w") as f:
+    json.dump(db_json, f, indent=4)

convert/generate_postgres_from_json.py

@@ -0,0 +1,72 @@
+import argparse
+import json
+import os
+import sys
+from datetime import datetime
+
+from fmp import FMP
+
+db_json = dict()
+fmp_model = FMP()
+CURR_DAY = datetime.now().strftime("%Y%m%d")
+
+
+# 1st (sql file)
+# 2nd (write to sql connection)
+def write_command(string):
+    with open("./init/{curr_day}_init_db.sql".format(curr_day=CURR_DAY), "w") as f:
+        f.write(string)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-m", "--model", required=True, help="JSON model file to seed")
+    args = parser.parse_args()
+    print(args)
+    if not os.path.isfile(os.path.abspath(args.model)):
+        raise ValueError("File: {0} doesn't exist".format(args.model))
+    return args
+
+
+def main(argv):
+    model_file = argv.model
+    with open(model_file, "r") as f:
+        data_model = json.load(f)
+
+    schema = "chi_{day}".format(day=CURR_DAY)
+    command = "create schema {schema};\n".format(schema=schema)
+    for table_name, attribute_dict in data_model.items():
+        command += "create table {schema}.{table_name} (\n".format(
+            schema=schema, table_name=table_name
+        )
+        features = list(attribute_dict.keys())
+        last_elem = features[-1]
+        for feature in features:
+
+            extra_param_string = ""
+
+            if attribute_dict[feature]["pk"]:
+                extra_param_string += "primary key"
+
+            # elif attribute_dict[feature]["fk"]:
+            #     extra_param_string += "foreign key"
+
+            command += "\t{attr_name} {attr_type} {extra_params}".format(
+                attr_name=feature,
+                attr_type=fmp_model.python_types_to_sql[
+                    attribute_dict[feature]["type"]
+                ],
+                extra_params=extra_param_string,
+            )
+            if feature != last_elem:
+                command += ",\n"
+            else:
+                command += "\n"
+
+        command += ");\n\n"
+
+    write_command(command)
+
+
+if __name__ == "__main__":
+    sys.exit(main(parse_args()))

convert/init/20200306_init_db.sql

@@ -0,0 +1,280 @@
+create schema chi_20200306;
+create table chi_20200306.clinics (
+	pk_clinic_id FLOAT primary key,
+	clinic_name VARCHAR ,
+	creation_date TIMESTAMP ,
+	creation_acct VARCHAR ,
+	modified_date TIMESTAMP ,
+	modified_acct VARCHAR 
+);
+
+create table chi_20200306.patients (
+	pk_patient_id FLOAT primary key,
+	creation_date TIMESTAMP ,
+	creation_acct VARCHAR ,
+	modified_date TIMESTAMP ,
+	modified_acct VARCHAR ,
+	last_name VARCHAR ,
+	first_name VARCHAR ,
+	gender VARCHAR ,
+	face_photo VARCHAR ,
+	birthdate VARCHAR ,
+	phone VARCHAR ,
+	town VARCHAR ,
+	unique_name VARCHAR ,
+	unique_ID_name VARCHAR ,
+	nickname VARCHAR ,
+	needs_review VARCHAR ,
+	notes_modification_status VARCHAR ,
+	fk_geonameid VARCHAR 
+);
+
+create table chi_20200306.visits (
+	pk_visit_id FLOAT primary key,
+	creation_date TIMESTAMP ,
+	creation_acct VARCHAR ,
+	modified_date TIMESTAMP ,
+	modified_acct VARCHAR ,
+	fk_patient_id FLOAT ,
+	fk_clinic_id FLOAT ,
+	visit_date TIMESTAMP ,
+	visit_age FLOAT ,
+	ob_pregnant VARCHAR ,
+	ob_last_menstrual_period VARCHAR ,
+	bp_systolic_blood_pressure FLOAT ,
+	bp_diastolic_blood_pressure FLOAT ,
+	bp_mean_arterial_pressure FLOAT ,
+	temp_in_fahrenheit FLOAT ,
+	pulse_rate FLOAT ,
+	weight_in_lbs FLOAT ,
+	height_in_inches FLOAT ,
+	glasses_given VARCHAR ,
+	sunglasses_given VARCHAR ,
+	dx_surgical_candidate VARCHAR ,
+	surgical_comments VARCHAR ,
+	visit_age_unit VARCHAR ,
+	chief_complaint VARCHAR ,
+	history_of_present_illness VARCHAR ,
+	visit_age_years FLOAT ,
+	exam_comments VARCHAR ,
+	tests_comments VARCHAR ,
+	tests_blood_sugar_result FLOAT ,
+	tests_pregnancy_result VARCHAR ,
+	tests_urine_glucose_result VARCHAR ,
+	tests_urine_protein_result VARCHAR ,
+	tests_urine_leukocytes_result VARCHAR ,
+	tests_urine_nitrites_result VARCHAR ,
+	ob_gravida FLOAT ,
+	ob_para FLOAT ,
+	ob_abortus FLOAT ,
+	ob_history VARCHAR ,
+	ob_wants_planning VARCHAR ,
+	albendazole_given VARCHAR ,
+	first_visit VARCHAR ,
+	known_med_allergies VARCHAR ,
+	meds_currently_taken VARCHAR ,
+	tests_pregnancy_given VARCHAR ,
+	tests_blood_sugar_given VARCHAR ,
+	tests_eye_exam_given VARCHAR ,
+	tests_urine_analysis_given VARCHAR ,
+	dx_asthma VARCHAR ,
+	dx_allergies VARCHAR ,
+	dx_anemia VARCHAR ,
+	dx_malnutrition VARCHAR ,
+	dx_diabetes VARCHAR ,
+	dx_pterygium VARCHAR ,
+	dx_cataract VARCHAR ,
+	dx_dry_eye VARCHAR ,
+	dx_conjunctivitis VARCHAR ,
+	dx_fever VARCHAR ,
+	dx_possible_malaria VARCHAR ,
+	dx_filariasis VARCHAR ,
+	dx_gastritis VARCHAR ,
+	dx_gerd VARCHAR ,
+	dx_gastroenteritis VARCHAR ,
+	dx_dysentery VARCHAR ,
+	dx_possible_cholera VARCHAR ,
+	dx_headache VARCHAR ,
+	dx_hernia VARCHAR ,
+	dx_hernia_type VARCHAR ,
+	dx_hypertension VARCHAR ,
+	dx_stroke VARCHAR ,
+	dx_angina VARCHAR ,
+	dx_chf VARCHAR ,
+	dx_musculoskeletal_pain VARCHAR ,
+	dx_otitis VARCHAR ,
+	dx_otitis_ear VARCHAR ,
+	dx_pneumonia VARCHAR ,
+	dx_sinusitis VARCHAR ,
+	dx_uri VARCHAR ,
+	dx_pregnant VARCHAR ,
+	dx_pregnant_weeks FLOAT ,
+	dx_atopic_derm VARCHAR ,
+	dx_fungal VARCHAR ,
+	dx_scabies VARCHAR ,
+	dx_impetigo VARCHAR ,
+	dx_tinea_capitus VARCHAR ,
+	dx_possible_std VARCHAR ,
+	dx_possible_tb VARCHAR ,
+	dx_vaginitis_candidal VARCHAR ,
+	dx_vaginitis_trich VARCHAR ,
+	dx_vaginitis_bv VARCHAR ,
+	dx_other VARCHAR ,
+	dx_other_text VARCHAR ,
+	tx_multivitamin VARCHAR ,
+	tx_pain_acetaminophen VARCHAR ,
+	tx_gi_omeprazole VARCHAR ,
+	tx_gi_pepto_bismol VARCHAR ,
+	tx_gi_cimetidine VARCHAR ,
+	tx_gi_tums VARCHAR ,
+	tx_gi_ors VARCHAR ,
+	tx_infections_albendazole VARCHAR ,
+	tx_infections_amoxicillin VARCHAR ,
+	tx_infections_azithromycin VARCHAR ,
+	tx_infections_bactrim VARCHAR ,
+	tx_infections_cipro VARCHAR ,
+	tx_infections_doxycycline VARCHAR ,
+	tx_infections_flagyl VARCHAR ,
+	tx_infections_fluconazole VARCHAR ,
+	tx_infections_griseofulvin VARCHAR ,
+	tx_infections_ivermectin VARCHAR ,
+	tx_infections_keflex VARCHAR ,
+	tx_infections_rocephin VARCHAR ,
+	tx_cardiac_amlodipine VARCHAR ,
+	tx_cardiac_aspirin VARCHAR ,
+	tx_cardiac_atenolol VARCHAR ,
+	tx_cardiac_hctz VARCHAR ,
+	tx_cardiac_lasix VARCHAR ,
+	tx_cardiac_enalapril VARCHAR ,
+	tx_topical_clotrimazole VARCHAR ,
+	tx_topical_hydrocortisone VARCHAR ,
+	tx_topical_silvadine VARCHAR ,
+	tx_topical_permethirine VARCHAR ,
+	tx_im_dexamethasone VARCHAR ,
+	tx_im_lasix VARCHAR ,
+	tx_misc_albuterol VARCHAR ,
+	tx_misc_benadryl VARCHAR ,
+	tx_misc_ferrous_sulfate VARCHAR ,
+	tx_misc_metformin VARCHAR ,
+	tx_misc_prednisolone VARCHAR ,
+	tx_misc_other VARCHAR ,
+	tx_misc_other_text VARCHAR ,
+	tx_comments VARCHAR ,
+	dx_uti VARCHAR ,
+	tx_multivitamin_varieties VARCHAR ,
+	tx_pain_acetaminophen_varieties VARCHAR ,
+	tx_infections_amoxicillin_varieties VARCHAR ,
+	tx_pain_ibuprofen VARCHAR ,
+	dx_comments VARCHAR ,
+	tx_infections_azithromycin_varieties VARCHAR ,
+	tx_infections_bactrim_varieties VARCHAR ,
+	tx_infections_keflex_varieties VARCHAR ,
+	tx_infections_rocephin_varieties VARCHAR ,
+	dx_vaginitis_general VARCHAR ,
+	dx_needs_to_see_dentist VARCHAR ,
+	dx_needs_to_see_eye_doctor VARCHAR ,
+	dx_healthy VARCHAR ,
+	provider VARCHAR ,
+	tx_saw_dentist VARCHAR ,
+	tx_saw_eye_doctor VARCHAR ,
+	tx_outside_referral VARCHAR ,
+	tx_outside_referral_text VARCHAR ,
+	tx_gi_zinc VARCHAR ,
+	tx_cardiac_methyldopa VARCHAR ,
+	tx_infections_augmentin VARCHAR ,
+	tx_infections_chloroquine VARCHAR ,
+	tx_infections_griseofuvin_varieties VARCHAR ,
+	tx_infections_nitrofurantoin VARCHAR ,
+	tx_topical_double_antibiotic VARCHAR ,
+	tx_misc_fluoxetine VARCHAR ,
+	tx_misc_ocp VARCHAR ,
+	tx_misc_phenytoin VARCHAR ,
+	dx_possible_parasites VARCHAR ,
+	tx_misc_iud VARCHAR ,
+	tests_id_test_given VARCHAR ,
+	tests_id_chlamydia_result VARCHAR ,
+	tests_id_hiv_result VARCHAR ,
+	tests_id_syphilis_result VARCHAR ,
+	tests_id_gonorrhea_result VARCHAR ,
+	tests_eye_exam_right_sphere VARCHAR ,
+	tests_eye_exam_right_cylinder VARCHAR ,
+	tests_eye_exam_right_axis VARCHAR ,
+	tests_eye_exam_right_bifocal VARCHAR ,
+	tests_eye_exam_left_sphere VARCHAR ,
+	tests_eye_exam_left_cylinder VARCHAR ,
+	tests_eye_exam_left_axis VARCHAR ,
+	tests_eye_exam_left_bifocal VARCHAR ,
+	tx_pain_ibuprofen_varieties VARCHAR ,
+	tx_misc_benadryl_varieties VARCHAR ,
+	tests_id_tb_result VARCHAR ,
+	tests_id_confirm_hiv_cd4 VARCHAR ,
+	tests_id_confirm_hiv_result VARCHAR ,
+	tests_id_confirm_hiv_arvt VARCHAR ,
+	tests_id_confirm_hiv_enrolled VARCHAR ,
+	tests_id_confirm_hiv_problems VARCHAR ,
+	tests_id_confirm_tb_result VARCHAR ,
+	tests_id_confirm_tb_comments VARCHAR ,
+	tests_id_confirm_hiv_enrolled_at VARCHAR ,
+	dx_depression VARCHAR ,
+	clinic_trip VARCHAR ,
+	tx_chw_referral VARCHAR ,
+	tx_misc_depo VARCHAR ,
+	tx_misc_paroxetine VARCHAR ,
+	tx_misc_keppra VARCHAR ,
+	dx_totalcount FLOAT ,
+	tx_totalcount FLOAT ,
+	tx_given_glasses VARCHAR ,
+	tx_misc_labetolol VARCHAR ,
+	tx_misc_carbamazepine VARCHAR ,
+	tx_misc_terazosin VARCHAR ,
+	tx_gi_ranitidine VARCHAR ,
+	tx_cardiac_lisinopril VARCHAR ,
+	tx_topical_triple_antibiotic VARCHAR ,
+	tx_misc_prednisone VARCHAR ,
+	clinic VARCHAR ,
+	notes_modification_status VARCHAR 
+);
+
+create table chi_20200306.diagnoses (
+	pk_diagnosis_id FLOAT primary key,
+	diagnosis_name VARCHAR ,
+	pk_diagnosis_key VARCHAR primary key,
+	creation_date TIMESTAMP ,
+	creation_acct VARCHAR ,
+	modified_date TIMESTAMP ,
+	modified_acct VARCHAR 
+);
+
+create table chi_20200306.diagnoses_made (
+	pk_diagnoses_made FLOAT primary key,
+	fk_visit_id FLOAT ,
+	fk_diagnosis_key VARCHAR ,
+	creation_date TIMESTAMP ,
+	creation_acct VARCHAR ,
+	modified_date TIMESTAMP ,
+	modified_acct VARCHAR 
+);
+
+create table chi_20200306.meds (
+	pk_med_id FLOAT primary key,
+	med_name VARCHAR ,
+	creation_date TIMESTAMP ,
+	creation_acct VARCHAR ,
+	modified_date TIMESTAMP ,
+	modified_acct VARCHAR ,
+	pk_med_key VARCHAR primary key,
+	med_type VARCHAR 
+);
+
+create table chi_20200306.meds_dispensed (
+	pk_meds_dispensed FLOAT primary key,
+	fk_visit_id FLOAT ,
+	fk_med_key VARCHAR ,
+	creation_date TIMESTAMP ,
+	creation_acct VARCHAR ,
+	modified_date TIMESTAMP ,
+	modified_acct VARCHAR ,
+	dispensed_total FLOAT ,
+	med_dosage VARCHAR 
+);
+