medicaid.cwl

  1#!/usr/bin/env cwl-runner
  2### Full Medicaid Processing Pipeline
  3#  Copyright (c) 2021. Harvard University
  4#
  5#  Developed by Research Software Engineering,
  6#  Faculty of Arts and Sciences, Research Computing (FAS RC)
  7#  Author: Michael A Bouzinier
  8#
  9#  Licensed under the Apache License, Version 2.0 (the "License");
 10#  you may not use this file except in compliance with the License.
 11#  You may obtain a copy of the License at
 12#
 13#         http://www.apache.org/licenses/LICENSE-2.0
 14#
 15#  Unless required by applicable law or agreed to in writing, software
 16#  distributed under the License is distributed on an "AS IS" BASIS,
 17#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 18#  See the License for the specific language governing permissions and
 19#  limitations under the License.
 20#
 21
 22cwlVersion: v1.2
 23class: Workflow
 24
 25requirements:
 26  SubworkflowFeatureRequirement: {}
 27  StepInputExpressionRequirement: {}
 28  InlineJavascriptRequirement: {}
 29
 30doc: |
 31  This workflow ingests Medicaid data, provided by
 32  Centers for Medicare & Medicaid Services (CMS)
 33  to researches. The expected input format is
 34  Medicaid Analytic eXtract (MAX) data.
 35
 36  The workflow parses File transfer summary (FTS) files,
 37  loads the raw data into a PostgreSQL DBMS and then processes
 38  the data to prepare it for using by NSAPH researches.
 39  See [documentation](../Medicaid.md) for detailed
 40  information.
 41
 42inputs:
 43  database:
 44    type: File
 45    doc: Path to database connection file, usually database.ini
 46  connection_name:
 47    type: string
 48    doc: The name of the section in the database.ini file
 49  input:
 50    type: Directory
 51    doc: |
 52      A path to directory, containing unpacked CMS
 53      files. The tool will recursively look for data files
 54      according to provided pattern
 55
 56steps:
 57  states:
 58    run: ensure_resource.cwl
 59    doc: |
 60      Ensures the presence of `us_states` table in the database.
 61      The table contains mapping between state names, ids
 62      (two letter abbreviations), FIPS codes and
 63      [ISO-3166-2 codes](https://en.wikipedia.org/wiki/ISO_3166-2)
 64    in:
 65      database: database
 66      connection_name: connection_name
 67      table:
 68        valueFrom: "us_states"
 69    out: [log]
 70  iso:
 71    run: ensure_resource.cwl
 72    doc: |
 73      Ensures the presence of `us_iso` table in the database.
 74      The table provides a mapping between states, counties and zip
 75      codes. It contains FIPS and
 76      [ISO-3166-2 codes](https://en.wikipedia.org/wiki/ISO_3166-2)
 77    in:
 78      database: database
 79      connection_name: connection_name
 80      table:
 81        valueFrom: "us_iso"
 82    out: [log]
 83  fts:
 84    run: parse_fts.cwl
 85    in:
 86      input: input
 87      output:
 88        valueFrom: cms.yaml
 89    out: [log, model, errors]
 90
 91  load_ps:
 92    run: ingest_cms.cwl
 93    doc: Loads Patient Summaries
 94    in:
 95      registry: fts/model
 96      domain:
 97        valueFrom: "cms"
 98      table:
 99        valueFrom: "ps"
100      input: input
101      database: database
102      connection_name: connection_name
103      incremental:
104        valueFrom: $(true)
105    out:
106      - reset_log
107      - create_log
108      - index_log
109      - vacuum_log
110      - reset_err
111      - create_err
112      - index_err
113      - vacuum_err
114
115  load_ip:
116    run: ingest_cms.cwl
117    doc: Loads inpatient admissions
118    in:
119      depends_on: load_ps/vacuum_log
120      registry: fts/model
121      domain:
122        valueFrom: "cms"
123      table:
124        valueFrom: "ip"
125      input: input
126      database: database
127      connection_name: connection_name
128      incremental:
129        valueFrom: $(true)
130    out:
131      - reset_log
132      - create_log
133      - index_log
134      - vacuum_log
135      - reset_err
136      - create_err
137      - index_err
138      - vacuum_err
139
140  create_beneficiaries:
141    run: matview.cwl
142    doc: Creates `Beneficiaries` Table
143    in:
144      depends_on: load_ip/vacuum_log
145      table:
146        valueFrom: "beneficiaries"
147      database: database
148      connection_name: connection_name
149    out:
150      - create_log
151      - index_log
152      - vacuum_log
153      - create_err
154      - index_err
155      - vacuum_err
156
157  create_monthly_view:
158    run: matview.cwl
159    doc: Creates internally used `Monthly View`
160    in:
161      depends_on: create_beneficiaries/vacuum_log
162      table:
163        valueFrom: "monthly"
164      database: database
165      connection_name: connection_name
166    out:
167      - create_log
168      - index_log
169      - vacuum_log
170      - create_err
171      - index_err
172      - vacuum_err
173
174  create_enrollments:
175    run: matview.cwl
176    doc: Creates `Enrollment` Table
177    in:
178      depends_on: create_monthly_view/vacuum_log
179      table:
180        valueFrom: "enrollments"
181      database: database
182      connection_name: connection_name
183    out:
184      - create_log
185      - index_log
186      - vacuum_log
187      - create_err
188      - index_err
189      - vacuum_err
190
191  prepare_eligibility:
192    run: create.cwl
193    doc: |
194      Creates `_Eligibility` View,
195      prerequisit for creation of Eligibility table
196    in:
197      depends_on: create_enrollments/vacuum_log
198      table:
199        valueFrom: "_eligibility"
200      database: database
201      connection_name: connection_name
202    out:
203      - log
204      - errors
205
206  create_eligibility:
207    run: matview.cwl
208    doc: Creates `Eligibility` Table
209    in:
210      depends_on: prepare_eligibility/log
211      table:
212        valueFrom: "eligibility"
213      database: database
214      connection_name: connection_name
215    out:
216      - create_log
217      - index_log
218      - vacuum_log
219      - create_err
220      - index_err
221      - vacuum_err
222
223  load_admissions:
224    run: ingest_cms.cwl
225    doc: Processes and loads inpatient admissions
226    in:
227      depends_on: create_eligibility/vacuum_log
228      domain:
229        valueFrom: "medicaid"
230      table:
231        valueFrom: "admissions"
232      input: input
233      database: database
234      connection_name: connection_name
235      incremental:
236        valueFrom: $(true)
237    out:
238      - reset_log
239      - create_log
240      - index_log
241      - vacuum_log
242      - reset_err
243      - create_err
244      - index_err
245      - vacuum_err
246
247outputs:
248  resource1_log:
249    type: File
250    outputSource: states/log
251  resource2_log:
252    type: File
253    outputSource: iso/log
254  parse_log:
255    type: File?
256    outputSource: fts/log
257
258  ps_reset_log:
259    type: File
260    outputSource: load_ps/reset_log
261  ps_create_log:
262    type: File
263    outputSource: load_ps/create_log
264  ps_index_log:
265    type: File
266    outputSource: load_ps/index_log
267  ps_vacuum_log:
268    type: File
269    outputSource: load_ps/vacuum_log
270  ip_reset_log:
271    type: File
272    outputSource: load_ip/reset_log
273  ip_create_log:
274    type: File
275    outputSource: load_ip/create_log
276  ip_index_log:
277    type: File
278    outputSource: load_ip/index_log
279  ip_vacuum_log:
280    type: File
281    outputSource: load_ip/vacuum_log
282
283  ben_create_log:
284    type: File
285    outputSource: create_beneficiaries/create_log
286  ben_index_log:
287    type: File
288    outputSource: create_beneficiaries/index_log
289  ben_vacuum_log:
290    type: File
291    outputSource: create_beneficiaries/vacuum_log
292  mnth_create_log:
293    type: File
294    outputSource: create_monthly_view/create_log
295  mnth_index_log:
296    type: File
297    outputSource: create_monthly_view/index_log
298  mnth_vacuum_log:
299    type: File
300    outputSource: create_monthly_view/vacuum_log
301  enrlm_create_log:
302    type: File
303    outputSource: create_enrollments/create_log
304  enrlm_index_log:
305    type: File
306    outputSource: create_enrollments/index_log
307  enrlm_vacuum_log:
308    type: File
309    outputSource: create_enrollments/vacuum_log
310  elgb_prepare_log:
311    type: File
312    outputSource: prepare_eligibility/log
313  elgb_create_log:
314    type: File
315    outputSource: create_eligibility/create_log
316  elgb_index_log:
317    type: File
318    outputSource: create_eligibility/index_log
319  elgb_vacuum_log:
320    type: File
321    outputSource: create_eligibility/vacuum_log
322
323  admissions_reset_log:
324    type: File
325    outputSource: load_admissions/reset_log
326  admissions_create_log:
327    type: File
328    outputSource: load_admissions/create_log
329  admissions_index_log:
330    type: File
331    outputSource: load_admissions/index_log
332  admissions_vacuum_log:
333    type: File
334    outputSource: load_admissions/vacuum_log
335
336## ERROR LOGS ###########################################
337
338  parse_err:
339    type: File
340    outputSource: fts/errors
341
342  ps_reset_err:
343    type: File
344    outputSource: load_ps/reset_err
345  ps_create_err:
346    type: File
347    outputSource: load_ps/create_err
348  ps_index_err:
349    type: File
350    outputSource: load_ps/index_err
351  ps_vacuum_err:
352    type: File
353    outputSource: load_ps/vacuum_err
354  ip_reset_err:
355    type: File
356    outputSource: load_ip/reset_err
357  ip_create_err:
358    type: File
359    outputSource: load_ip/create_err
360  ip_index_err:
361    type: File
362    outputSource: load_ip/index_err
363  ip_vacuum_err:
364    type: File
365    outputSource: load_ip/vacuum_err
366
367
368  ben_create_err:
369    type: File
370    outputSource: create_beneficiaries/create_err
371  ben_index_err:
372    type: File
373    outputSource: create_beneficiaries/index_err
374  ben_vacuum_err:
375    type: File
376    outputSource: create_beneficiaries/vacuum_err
377  mnth_create_err:
378    type: File
379    outputSource: create_monthly_view/create_err
380  mnth_index_err:
381    type: File
382    outputSource: create_monthly_view/index_err
383  mnth_vacuum_err:
384    type: File
385    outputSource: create_monthly_view/vacuum_err
386  enrlm_create_err:
387    type: File
388    outputSource: create_enrollments/create_err
389  enrlm_index_err:
390    type: File
391    outputSource: create_enrollments/index_err
392  enrlm_vacuum_err:
393    type: File
394    outputSource: create_enrollments/vacuum_err
395  elgb_create_err:
396    type: File
397    outputSource: create_eligibility/create_err
398  elgb_prepare_err:
399    type: File
400    outputSource: prepare_eligibility/errors
401  elgb_index_err:
402    type: File
403    outputSource: create_eligibility/index_err
404  elgb_vacuum_err:
405    type: File
406    outputSource: create_eligibility/vacuum_err
407  admissions_reset_err:
408    type: File
409    outputSource: load_admissions/reset_err
410  admissions_create_err:
411    type: File
412    outputSource: load_admissions/create_err
413  admissions_index_err:
414    type: File
415    outputSource: load_admissions/index_err
416  admissions_vacuum_err:
417    type: File
418    outputSource: load_admissions/vacuum_err