"""Shared configuration: paths, constants, and exploratory mappings.
All reusable lookup tables (FIPS→region, RUCA→urban_rural bucket, DRG→MDC,
APC→family) live here so the cleaning/analysis layers never duplicate logic.
The DRG/APC groupings are text-derived exploratory features, not official CMS
crosswalks.
"""
import re
from pathlib import Path
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
ROOT = Path(__file__).resolve().parents[1]
DATA_RAW = ROOT / "data" / "raw"
DATA_PROC = ROOT / "data" / "processed"
SITE_DIR = ROOT / "site"
SITE_DATA = SITE_DIR / "data"
SITE_CODE = SITE_DIR / "code"
for p in (DATA_PROC, SITE_DATA, SITE_CODE):
p.mkdir(parents=True, exist_ok=True)
# ---------------------------------------------------------------------------
# Rename map — standardize the cryptic CMS column names to readable ones.
# Applied identically to both inpatient and outpatient frames in clean.py.
# ---------------------------------------------------------------------------
COLUMN_RENAME = {
"Rndrng_Prvdr_CCN": "provider_ccn",
"Rndrng_Prvdr_Org_Name": "provider_name",
"Rndrng_Prvdr_St": "provider_street",
"Rndrng_Prvdr_City": "provider_city",
"Rndrng_Prvdr_State_Abrvtn": "state",
"Rndrng_Prvdr_State_FIPS": "state_fips",
"Rndrng_Prvdr_Zip5": "zip5",
"Rndrng_Prvdr_RUCA": "ruca_code",
"Rndrng_Prvdr_RUCA_Desc": "ruca_desc",
# service codes left longer to read naturally in tables
"DRG_Cd": "drg_code",
"DRG_Desc": "drg_desc",
"APC_Cd": "apc_code",
"APC_Desc": "apc_desc",
# inpatient numerics
"Tot_Dschrgs": "total_discharges",
"Avg_Submtd_Cvrd_Chrg": "avg_submitted_charge",
"Avg_Tot_Pymt_Amt": "avg_total_payment", # primary inpatient cost measure
"Avg_Mdcr_Pymt_Amt": "avg_medicare_payment",
# outpatient numerics
"Bene_Cnt": "bene_count",
"CAPC_Srvcs": "apc_services",
"Avg_Tot_Sbmtd_Chrgs": "avg_submitted_charge",
"Avg_Mdcr_Alowd_Amt": "avg_allowed_amount", # primary outpatient cost measure
"Avg_Mdcr_Pymt_Amt": "avg_medicare_payment",
"Outlier_Srvcs": "outlier_services",
"Avg_Mdcr_Outlier_Amt": "avg_outlier_amount",
}
# Numeric columns per dataset (against the *renamed* schema)
INPATIENT_NUMERIC = [
"total_discharges", "avg_submitted_charge",
"avg_total_payment", "avg_medicare_payment",
]
OUTPATIENT_NUMERIC = [
"bene_count", "apc_services", "avg_submitted_charge",
"avg_allowed_amount", "avg_medicare_payment",
"outlier_services", "avg_outlier_amount",
]
# ---------------------------------------------------------------------------
# Dataset periods, payment measures, and feature formulas
# ---------------------------------------------------------------------------
# The two CMS files cover different annual periods. Keep the distinction in
# generated metadata rather than silently treating both files as "2023".
DATASET_METADATA = {
"inpatient": {
"period": "FY2023",
"period_type": "fiscal year",
"payment_measure": {
"field": "avg_total_payment",
"source_column": "Avg_Tot_Pymt_Amt",
"label": "Average total payment",
"definition": "Average total payment for the inpatient DRG, including beneficiary and third-party amounts.",
"role": "primary payment denominator for the inpatient ratio",
},
"ratio": {
"field": "charge_to_total_payment_ratio",
"formula": "avg_submitted_charge / avg_total_payment",
"numerator": "avg_submitted_charge",
"denominator": "avg_total_payment",
"compatibility_alias": "charge_to_payment_ratio",
},
},
"outpatient": {
"period": "CY2023",
"period_type": "calendar year",
"payment_measure": {
"field": "avg_allowed_amount",
"source_column": "Avg_Mdcr_Alowd_Amt",
"label": "Average Medicare allowed amount",
"definition": "Average allowed amount, including the Medicare and beneficiary shares; CMS suppresses it for low-service rows.",
"role": "primary observed-cost/payment denominator for the outpatient ratio",
},
"ratio": {
"field": "charge_to_allowed_amount_ratio",
"formula": "avg_submitted_charge / avg_allowed_amount",
"numerator": "avg_submitted_charge",
"denominator": "avg_allowed_amount",
"compatibility_alias": "charge_to_payment_ratio",
},
},
}
COMPATIBILITY_RATIO_ALIAS = "charge_to_payment_ratio"
# ---------------------------------------------------------------------------
# Exploratory DRG/APC mapping disclosure and sanity checks
# ---------------------------------------------------------------------------
MAPPING_STATUS = "exploratory/text-derived"
MAPPING_METADATA = {
"mapping_status": MAPPING_STATUS,
"official_crosswalk_available_locally": False,
"source": "Observed DRG_Desc/APC_Desc text with local keyword rules and coarse numeric fallback for uncoded DRGs.",
"disclaimer": "These groupings are exploratory and are not an official CMS DRG-to-MDC or APC crosswalk.",
"modeling_policy": "Known-invalid DRG families are quarantined from predictor modeling; remaining families are exploratory features.",
}
# Observed descriptions for these DRGs expose known failures of the local
# keyword/fallback classifier. They remain visible in descriptive data, but do
# not enter mapping-dependent predictor fits until an official crosswalk is
# supplied.
DRG_MAPPING_QUARANTINE_REASONS = {
"896": "description contains a negative rehabilitation phrase that defeats a simple substring rule",
"897": "description contains a negative rehabilitation phrase that defeats a simple substring rule",
"919": "numeric fallback is not a reliable MDC assignment for complications-of-treatment descriptions",
"920": "numeric fallback is not a reliable MDC assignment for complications-of-treatment descriptions",
"921": "numeric fallback is not a reliable MDC assignment for complications-of-treatment descriptions",
"922": "short keyword matching can confuse NOSE with DIAGNOSES in observed descriptions",
"923": "short keyword matching can confuse NOSE with DIAGNOSES in observed descriptions",
}
def _normalise_drg_code(value) -> str | None:
"""Return a comparable three-character DRG code when possible."""
if value is None:
return None
text = str(value).strip()
if text.endswith(".0") and text[:-2].isdigit():
text = text[:-2]
return text.zfill(3) if text.isdigit() else text or None
def drg_mapping_quarantine_reason(drg_code, drg_desc="") -> str | None:
"""Return a known mapping failure reason, or ``None``."""
code = _normalise_drg_code(drg_code)
return DRG_MAPPING_QUARANTINE_REASONS.get(code)
# These are deliberately description-based sanity checks, not claims of an
# external authoritative crosswalk. The descriptions are present in the raw
# files and exercise the two known corrections.
MAPPING_VALIDATION_EXAMPLES = {
"drg": [
{
"code": "023",
"expected_mapping": "Nervous System",
"description_contains": "CRANIOTOMY",
},
{
"code": "003",
"expected_mapping": "Transplant / Pre-MDC",
"description_contains": "ECMO",
},
],
"apc": [
{
"code": "5471",
"expected_mapping": "Drug Infusion",
"description_contains": "DRUG INFUSION",
},
{
"code": "5072",
"expected_mapping": "Excision / Biopsy / Incision",
"description_contains": "EXCISION",
},
],
}
# ---------------------------------------------------------------------------
# FIPS-2 → Census region
# Source: U.S. Census Bureau — 4 regions, 9 divisions
# ---------------------------------------------------------------------------
FIPS_TO_REGION = {
# Northeast
"09": "Northeast", "23": "Northeast", "50": "Northeast",
"33": "Northeast", "44": "Northeast", "25": "Northeast",
"34": "Northeast", "36": "Northeast", "42": "Northeast",
# Midwest
"17": "Midwest", "18": "Midwest", "26": "Midwest", "39": "Midwest",
"55": "Midwest", "19": "Midwest", "20": "Midwest", "27": "Midwest",
"29": "Midwest", "31": "Midwest", "38": "Midwest", "46": "Midwest",
# South
"10": "South", "11": "South", "12": "South", "13": "South",
"24": "South", "37": "South", "45": "South", "51": "South",
"54": "South", "01": "South", "21": "South", "28": "South",
"47": "South", "05": "South", "22": "South", "40": "South",
"48": "South",
# West
"04": "West", "08": "West", "16": "West", "30": "West",
"32": "West", "35": "West", "49": "West", "56": "West",
"02": "West", "15": "West", "06": "West", "41": "West",
"53": "West",
# DC — administrative convenience; small N
"11": "South",
}
# ---------------------------------------------------------------------------
# RUCA code → urban / micropolitan / rural bucket
# USDA RUCA taxonomy: codes 1-3 metro, 4-6 micro, 7-10 small town / rural
# ---------------------------------------------------------------------------
def ruca_to_urban_rural(code) -> str:
if code in (None, "", "Unknown"):
return "Unknown"
try:
c = float(code)
except (TypeError, ValueError):
return "Unknown"
if c <= 3:
return "Urban"
if c <= 6:
return "Micropolitan"
if c <= 10:
return "Rural"
return "Unknown"
# ---------------------------------------------------------------------------
# DRG → Major Diagnostic Category (MDC)
# MS-DRG numbering is not strictly sequential by MDC (CMS periodically inserts
# new DRGs into existing ranges), so we use a description-keyword classifier
# with a coarse numeric fallback for rare descriptions that do not match. This
# is more transparent than presenting a locally invented table as official.
# ---------------------------------------------------------------------------
_DRG_KEYWORDS = [
# (substring, MDC label) — first match wins; order matters
("REHABILITATION", "Rehabilitation"),
("BURNS", "Burns"),
("TRAUMA", "Trauma / Injury"),
("HIV", "HIV / AIDS"),
("MENTAL", "Mental Health"),
("DEPRESSION", "Mental Health"),
("PSYCH", "Mental Health"),
("SUBSTANCE", "Substance Use"),
("ALCOHOL", "Substance Use"),
("POISON", "Injury / Poisoning"),
("TOXIC", "Injury / Poisoning"),
("DRUG", "Substance Use"),
("PREGNANCY", "Pregnancy & Childbirth"),
("CHILD BIRTH", "Pregnancy & Childbirth"),
("DELIVERY", "Pregnancy & Childbirth"),
("ABORTION", "Pregnancy & Childbirth"),
("PUERPERIUM", "Pregnancy & Childbirth"),
("NEWBORN", "Newborn & Neonate"),
("NEONATE", "Newborn & Neonate"),
("TRANSPLANT", "Transplant / Pre-MDC"),
("TRACHEOSTOMY", "Transplant / Pre-MDC"),
("ECMO", "Transplant / Pre-MDC"),
# MDC 4 - Respiratory
("RESPIRATORY", "Respiratory"),
("PULMONARY", "Respiratory"),
("PNEUMONIA", "Respiratory"),
("COPD", "Respiratory"),
("ASTHMA", "Respiratory"),
("BRONCHITIS", "Respiratory"),
("PLEURAL", "Respiratory"),
("LUNG", "Respiratory"),
("VENTIL", "Respiratory"),
("PNEUMOTHORAX", "Respiratory"),
("PULMONARY EMBOLISM", "Respiratory"),
# MDC 5 - Circulatory
("CARDIAC", "Circulatory"),
("HEART", "Circulatory"),
("CORONARY", "Circulatory"),
("MYOCARDIAL", "Circulatory"),
("ARRHYTHMIA", "Circulatory"),
("HYPERTENSION", "Circulatory"),
("CIRCULATORY", "Circulatory"),
("ANGINA", "Circulatory"),
("VALVULAR", "Circulatory"),
("PACEMAKER", "Circulatory"),
("DEFIBRILL", "Circulatory"),
("AORTIC", "Circulatory"),
("STROKE", "Circulatory"),
# MDC 6/7 - Digestive + Hepatobiliary
("GASTROINTEST", "Digestive"),
("DIGESTIVE", "Digestive"),
("ESOPHAG", "Digestive"),
("STOMACH", "Digestive"),
("INTESTINE", "Digestive"),
("ABDOMEN", "Digestive"),
("APPENDIX", "Digestive"),
("HERNIA", "Digestive"),
("GALLBLADDER", "Hepatobiliary & Pancreas"),
("BILIARY", "Hepatobiliary & Pancreas"),
("LIVER", "Hepatobiliary & Pancreas"),
("HEPATIC", "Hepatobiliary & Pancreas"),
("PANCREA", "Hepatobiliary & Pancreas"),
# MDC 8 - Musculoskeletal
("HIP", "Musculoskeletal"),
("KNEE", "Musculoskeletal"),
("JOINT", "Musculoskeletal"),
("MUSCULOSKELETAL", "Musculoskeletal"),
("SPINE", "Musculoskeletal"),
("SPINAL", "Musculoskeletal"),
("LIMB", "Musculoskeletal"),
("EXTREMITY", "Musculoskeletal"),
("FRACTURE", "Musculoskeletal"),
("BONE", "Musculoskeletal"),
("TENDON", "Musculoskeletal"),
("MUSCLE", "Musculoskeletal"),
("FEMUR", "Musculoskeletal"),
# MDC 9 - Skin / Breast
("SKIN", "Skin & Subcutaneous Tissue"),
("BREAST", "Skin / Breast"),
("LYMPHATIC", "Skin / Breast"),
("DEBRIDEMENT", "Skin & Subcutaneous Tissue"),
("ULCER", "Skin & Subcutaneous Tissue"),
# MDC 10 - Endocrine
("ENDOCRINE", "Endocrine / Metabolic"),
("NUTRITIONAL", "Endocrine / Metabolic"),
("METABOLIC", "Endocrine / Metabolic"),
("DIABETES", "Endocrine / Metabolic"),
("THYROID", "Endocrine / Metabolic"),
("PITUITARY", "Endocrine / Metabolic"),
("ADRENAL", "Endocrine / Metabolic"),
# MDC 11 - Kidney / Urinary
("KIDNEY", "Kidney & Urinary Tract"),
("RENAL", "Kidney & Urinary Tract"),
("URINARY", "Kidney & Urinary Tract"),
("URETER", "Kidney & Urinary Tract"),
("BLADDER", "Kidney & Urinary Tract"),
("NEPHROT", "Kidney & Urinary Tract"),
("DIALYSIS", "Kidney & Urinary Tract"),
("TRANSURETHRAL", "Kidney & Urinary Tract"),
("PROSTATECTOMY", "Kidney & Urinary Tract"),
# MDC 12/13 - Reproductive
("PROSTATE", "Male Reproductive"),
("TESTES", "Male Reproductive"),
("MALE PELVIC", "Male Reproductive"),
("UTERUS", "Female Reproductive"),
("OVARY", "Female Reproductive"),
("FALLOPIAN", "Female Reproductive"),
("CERVICAL", "Female Reproductive"),
("GYNECOLOG", "Female Reproductive"),
# MDC 1 - Nervous
("INTRACRANIAL", "Nervous System"),
("CRANIAL", "Nervous System"),
("CRANIOTOMY", "Nervous System"),
("BRAIN", "Nervous System"),
("NERVOUS", "Nervous System"),
("DEGENERATIVE NERVOUS", "Nervous System"),
("SEIZURE", "Nervous System"),
("HEADACHE", "Nervous System"),
("MULTIPLE SCLEROSIS", "Nervous System"),
("ISCHEMIA", "Nervous System"),
("CEREBROVASC", "Nervous System"),
("TRANSIENT ISCHEM", "Nervous System"),
("SPINAL CORD", "Nervous System"),
# MDC 2 - Eye
("INTRAOCULAR", "Eye"),
("RETINAL", "Eye"),
("CORNEAL", "Eye"),
("CATARACT", "Eye"),
("OCULAR", "Eye"),
("EYE", "Eye"),
("VITREOUS", "Eye"),
# MDC 3 - ENT
("EAR", "Ear, Nose, Mouth & Throat"),
("NOSE", "Ear, Nose, Mouth & Throat"),
("SINUS", "Ear, Nose, Mouth & Throat"),
("MASTOID", "Ear, Nose, Mouth & Throat"),
("OTITIS", "Ear, Nose, Mouth & Throat"),
("LARYNX", "Ear, Nose, Mouth & Throat"),
("VESTIBULAR", "Ear, Nose, Mouth & Throat"),
("TONSIL", "Ear, Nose, Mouth & Throat"),
("ADENOID", "Ear, Nose, Mouth & Throat"),
("SLEEP APNEA", "Ear, Nose, Mouth & Throat"),
("EPISTAXIS", "Ear, Nose, Mouth & Throat"),
("MOUTH", "Ear, Nose, Mouth & Throat"),
("DENTAL", "Ear, Nose, Mouth & Throat"),
("THROAT", "Ear, Nose, Mouth & Throat"),
("SALIVARY", "Ear, Nose, Mouth & Throat"),
# MDC 16/17 - Blood / Myeloproliferative
("BLOOD", "Blood & Myeloproliferative"),
("ANEMIA", "Blood & Myeloproliferative"),
("COAGULATION", "Blood & Myeloproliferative"),
("SICKLE", "Blood & Myeloproliferative"),
("LEUKEMIA", "Blood & Myeloproliferative"),
("LYMPHOMA", "Blood & Myeloproliferative"),
("MYELOPROLIF", "Blood & Myeloproliferative"),
# MDC 18 - Infectious
("SEPTICEMIA", "Infectious & Parasitic"),
("SEPTIC", "Infectious & Parasitic"),
("INFECTION", "Infectious & Parasitic"),
("INFECTIOUS", "Infectious & Parasitic"),
# MDC 21 - Injuries/Poisoning
("INJURY", "Injury / Poisoning"),
("WOUND", "Injury / Poisoning"),
]
def drg_to_mdc(drg_code, drg_desc="") -> str:
"""Map an MS-DRG to an exploratory MDC-like family.
Description text is checked before the numeric fallback. In particular,
low-numbered DRGs are not all treated as Pre-MDC: DRG 023's craniotomy
description maps to the nervous-system family, while an ECMO description
can still map to the explicitly labelled Pre-MDC family.
"""
try:
d = int(drg_code)
except (TypeError, ValueError):
return "Unknown"
if drg_mapping_quarantine_reason(drg_code, drg_desc):
return "Other / Unclassified"
# Keyword match — mostly substring search, with word boundaries for short
# tokens that otherwise match inside unrelated words such as DIAGNOSES.
s = (drg_desc or "").upper()
for kw, mdc in _DRG_KEYWORDS:
if kw in {"EAR", "EYE", "HIP", "KNEE", "LUNG", "BONE", "NOSE", "MOUTH", "THROAT"}:
matched = re.search(rf"\b{re.escape(kw)}\b", s) is not None
else:
matched = kw in s
if matched:
return mdc
# Fallback: keep a coarse numeric partitioner for rare uncoded DRGs
if 20 <= d <= 103:
return "Nervous System"
if 110 <= d <= 196:
return "Eye"
if 200 <= d <= 303:
return "Ear, Nose, Mouth & Throat"
if 310 <= d <= 395:
return "Respiratory"
if 405 <= d <= 460:
return "Circulatory"
if 462 <= d <= 625:
# 462-469 musculoskeletal surgical
return "Musculoskeletal"
if 626 <= d <= 660:
return "Digestive"
if 676 <= d <= 700:
return "Hepatobiliary & Pancreas"
if 710 <= d <= 765:
return "Skin / Breast"
if 768 <= d <= 841:
return "Kidney & Urinary Tract"
if 842 <= d <= 897:
return "Endocrine / Metabolic"
if 901 <= d <= 925:
return "Male Reproductive"
if 940 <= d <= 959:
return "Pregnancy & Childbirth"
if 960 <= d <= 982:
return "Injury / Poisoning"
if 983 <= d <= 999:
return "Trauma / Injury"
return "Other / Unclassified"
# ---------------------------------------------------------------------------
# APC → Family
# Derived from `APC_Desc` keyword patterns observed in the dataset.
# ---------------------------------------------------------------------------
def apc_to_family(desc: str) -> str:
"""Map an APC description to an exploratory higher-level family."""
if not desc or not isinstance(desc, str):
return "Unknown"
s = desc.lower()
if "excision" in s or "biopsy" in s or "incision and drainage" in s:
return "Excision / Biopsy / Incision"
if "breast" in s or "lymphatic" in s:
return "Breast / Lymphatic Surgery"
if "musculoskeletal" in s:
return "Musculoskeletal"
if "airway endoscopy" in s:
return "Airway Endoscopy"
if " ENT" in desc or s.startswith("ent"):
return "ENT"
if "cochlear" in s:
return "ENT"
if "vascular procedures" in s:
return "Vascular"
if "endovascular" in s:
return "Endovascular"
if "electrophysiologic" in s:
return "Electrophysiologic"
if "pacemaker" in s:
return "Pacemaker"
if "icd" in s:
return "ICD"
if "blood product" in s:
return "Blood Product Exchange"
if "upper gi" in s:
return "Upper GI"
if "lower gi" in s:
return "Lower GI"
if "complex gi" in s:
return "Complex GI"
if "abdominal" in s or "peritoneal" in s or "biliary" in s:
return "Abdominal / Peritoneal / Biliary"
if "laparoscopy" in s:
return "Laparoscopy"
if "urology" in s:
return "Urology"
if "gynecologic" in s:
return "Gynecologic"
if "nerve procedures" in s:
return "Nerve Procedures"
if "drug infusion" in s:
return "Drug Infusion"
if "neurostimulator" in s:
return "Neurostimulator"
if "intraocular" in s or "extraocular" in s:
return "Eye Procedures"
if "radiation therapy" in s:
return "Radiation Therapy"
if "ancillary" in s:
return "Ancillary"
if "observation" in s:
return "Observation"
if "implantation" in s:
return "Device Implantation"
return "Other"
# ---------------------------------------------------------------------------
# Thresholds
# ---------------------------------------------------------------------------
MIN_PROVIDERS_PER_SERVICE = 30 # services need ≥30 providers to be statistically meaningful
IQR_MULTIPLIER = 1.5
ZSCORE_MODERATE = 2.0
ZSCORE_EXTREME = 3.0
DEFAULT_SEED = 42