<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<?covid-19-tdm?>
<article article-type="article-commentary" dtd-version="2.0" xmlns:xlink="http://www.w3.org/1999/xlink">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIRxMed</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIRx Med</journal-id>
      <journal-title>JMIRx Med</journal-title>
      <issn pub-type="epub">2563-6316</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v2i4e34081</article-id>
      <article-id pub-id-type="pmid"/>
      <article-id pub-id-type="doi">10.2196/34081</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Authors’ Response to Peer Reviews</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Authors’ Response to Peer Reviews</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Author’s Response to Peer Reviews of “Machine Learning–Based Prediction of COVID-19 Mortality With Limited Attributes to Expedite Patient Prognosis and Triage: Retrospective Observational Study”</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Meinert</surname>
            <given-names>Edward</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Doyle</surname>
            <given-names>Riccardo</given-names>
          </name>
          <degrees>BASc, MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Stuart Ltd</institution>
            <addr-line>2 London Wall Place</addr-line>
            <addr-line>Barbican</addr-line>
            <addr-line>London, EC2Y 5AU</addr-line>
            <country>United Kingdom</country>
            <phone>44 7380400958</phone>
            <email>r.doyle.edu@gmail.com</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-8975-2273</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Stuart Ltd</institution>
        <addr-line>London</addr-line>
        <country>United Kingdom</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Riccardo Doyle <email>r.doyle.edu@gmail.com</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <season>Oct-Dec</season>
        <year>2021</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>15</day>
        <month>10</month>
        <year>2021</year>
      </pub-date>
      <volume>2</volume>
      <issue>4</issue>
      <elocation-id>e34081</elocation-id>
      <history>
        <date date-type="received">
          <day>5</day>
          <month>10</month>
          <year>2021</year>
        </date>
        <date date-type="accepted">
          <day>5</day>
          <month>10</month>
          <year>2021</year>
        </date>
      </history>
      <copyright-statement>©Riccardo Doyle. Originally published in JMIRx Med (https://med.jmirx.org), 15.10.2021.</copyright-statement>
      <copyright-year>2021</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIRx Med, is properly cited. The complete bibliographic information, a link to the original publication on https://med.jmirx.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://med.jmirx.org/2021/4/e34081" xlink:type="simple"/>
      <related-article related-article-type="companion" id="preprint21253984" ext-link-type="doi" xlink:href="https://doi.org/10.1101/2021.03.21.21253984" vol="1" page="21253984" xlink:title="Preprint (MedRxiv):" xlink:type="simple">https://www.medrxiv.org/content/10.1101/2021.03.21.21253984v2</related-article>
      <related-article related-article-type="companion" id="preprint29392" ext-link-type="doi" xlink:href="https://doi.org/10.2196/preprints.29392" vol="2" page="e29392" xlink:title="Preprint (JMIR Preprints):" xlink:type="simple">https://preprints.jmir.org/preprint/29392</related-article>
      <related-article related-article-type="companion" id="v2i4e34082" ext-link-type="doi" xlink:href="10.2196/34082" vol="2" page="e34082" xlink:title="Peer-Review Report by Sebastian Daniel Boie (Reviewer DD):" xlink:type="simple">https://med.jmirx.org/2021/4/e34082/</related-article>
      <related-article related-article-type="companion" id="v2i4e34083" ext-link-type="doi" xlink:href="10.2196/34083" vol="2" page="e34083" xlink:title="Peer-Review Report by Victor Moquillaza Alcántara (Reviewer EB):" xlink:type="simple">https://med.jmirx.org/2021/4/e34083/</related-article>
      <related-article related-article-type="companion" id="v2i4e29392" ext-link-type="doi" xlink:href="10.2196/29392" vol="2" page="e29392" xlink:title="Published Article:" xlink:type="simple">https://med.jmirx.org/2021/4/e29392/</related-article>
      <kwd-group>
        <kwd>COVID-19</kwd>
        <kwd>coronavirus</kwd>
        <kwd>medical informatics</kwd>
        <kwd>machine learning</kwd>
        <kwd>artificial intelligence</kwd>
        <kwd>dimensionality reduction</kwd>
        <kwd>automation</kwd>
        <kwd>model development</kwd>
        <kwd>prediction</kwd>
        <kwd>hospital</kwd>
        <kwd>resource management</kwd>
        <kwd>mortality</kwd>
        <kwd>prognosis</kwd>
        <kwd>triage</kwd>
        <kwd>comorbidities</kwd>
        <kwd>public data</kwd>
        <kwd>epidemiology</kwd>
        <kwd>pre-existing conditions</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <p>
      <italic>This is the author’s response to peer-review reports for “Machine Learning–Based Prediction of COVID-19 Mortality With Limited Attributes to Expedite Patient Prognosis and Triage: Retrospective Observational Study.”</italic>
    </p>
    <sec>
      <title>Round 1 Review</title>
      <sec>
        <title>Reviewer DD</title>
        <p>More detail is provided in the responses to individual comments [<xref ref-type="bibr" rid="ref1">1</xref>], but for general context, to increase originality, the revised manuscript [<xref ref-type="bibr" rid="ref2">2</xref>] now focuses more heavily on the impact of feature reduction on model performance rather than model performance as a standalone finding. The original reduction method, mutual information, is complemented by chi-square reduction, and comparisons between the impact of each were made, highlighting the need for different reduction methods to be tested as part of model tuning. Additional points were added to the <italic>Discussion</italic> stating that comparable models drawing from much richer feature sets performed comparably to our reduced ones and that large amounts of explanatory power can be captured by even a single variable, with the ultimate goal of reducing the number of variables, and consequently the tests and imaging, needed before models can be used in a hospital setting.</p>
        <p>1. Mutual information was used due to the mixture of categorical and continuous variables, with a large presence of the former. A general equation for mutual information, which is the criterion used for feature selection, was provided. Variables were not binned but rather modelled through a k-nearest neighbors estimation approach; this was mentioned in the study, and the relevant source paper was cited for further detail. Software packages used (methods from Python’s sklearn library) were mentioned explicitly in the methodology.</p>
        <p>2. Features are a subset/extraction of the original feature set, not a transformation/combination. A section was added to the <italic>Results</italic> section detailing the 7 most salient features selected via mutual information.</p>
        <p>3. This would be a productive comparison; however, the reason it was not performed is due to data limitations. As outlined in the original paper, the 5121-patient data set has an extremely small proportion of patients affected by pre-existing conditions, meaning that keeping those features and training a 5121 patient model on age + comorbidities and comparing it to the full 212-patient data set would really simply be a comparison of the impact of age in the 5121-patient model against the full features in the 212-patient model, given that co-morbidity data is largely absent (and vastly underrepresented) in the 5121-patient data set. However, from other helpful revision comments, a feature importance table using mutual information was provided in the <italic>Results</italic> section; it shows that with the exception of fever, symptoms do not seem to play a high-importance role in prediction and do not feature in the top 7 explanatory variables.</p>
        <p>Additionally, to further facilitate comparisons between data sets and feature reductions, only the 212-patient data set was retained in the study.</p>
        <p>4. 95% confidence intervals were added to the result tables for all sensitivity, specificity, accuracy, and area under the curve (AUC) findings.</p>
        <p>5. A paragraph has been added to the <italic>Discussion</italic> section briefly comparing the 7 features extracted in our study using mutual information to the most salient features from the proposed paper, finding substantial overlap, particularly with fever and pneumonia as high-value features.</p>
        <p>6. The parameters used were reported in the <italic>Results</italic> section.</p>
        <p>7. The date on which the data were accessed was added to the relevant data section in the methodology. Detailing the exact breakdown of samples in training and testing over multiple iterations of sample splitting and dimensionality reduction seems excessive, especially considering that the sample is small and retrievable and the methodology (3-fold cross-validation coupled with simple classifiers) is easily reproducible.</p>
        <p>8. This was poor wording on the study’s end; it was intended to state that receiver operating characteristic (ROC) curves will be produced in order to obtain numerical AUC estimates, but the ROC plots were never meant to be graphically reported in the study. The original sentence was removed from the paper to avoid confusion.</p>
      </sec>
      <sec>
        <title>Reviewer EB</title>
        <p>1. Noted; the suggestion [<xref ref-type="bibr" rid="ref3">3</xref>] has been implemented. An in-depth review of existing equipment, public spending, and staff shortage limitations prior to the COVID-19 pandemic was provided with examples from around the world, with additional indicators of strain following the pandemic, as well as studies directly linking shortage of resources to worse patient outcomes, therefore justifying the need for better resource management. The primary management tool proposed in the study is the introduction of predictive modelling for better triage, providing potential benefits to “pre-allocation or local hospital transfer of life saving equipment, quantifying the need for further diagnostics or early treatment and directing limited staff attention and resources toward highest risk patients.” All condensed points in this response can be found in expanded form in the introduction of the study.</p>
        <p>2. In the new <italic>Discussion</italic> section, a paragraph has been added regarding the real-world use cases of the models explored in the study.</p>
        <p>3. Noted; a more streamlined and direct objective has been included in the new Abstract.</p>
        <p>4. Noted; this has now been rectified.</p>
        <p>5. Noted; this has now been rectified.</p>
        <p>6. This refers to the Area Under the Curve, and a footnote has now been added.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AUC</term>
          <def>
            <p>area under the curve</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">ROC</term>
          <def>
            <p>receiver operating characteristic</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Boie</surname>
              <given-names>SD</given-names>
            </name>
          </person-group>
          <article-title>Peer review of "Machine learning–based prediction of COVID-19 mortality with limited attributes to expedite patient prognosis and triage: retrospective observational study"</article-title>
          <source>JMIRx Med</source>
          <year>2021</year>
          <volume>2</volume>
          <issue>4</issue>
          <fpage>e34082</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://med.jmirx.org/2021/4/e34082/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/34082</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Doyle</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Machine learning–based prediction of COVID-19 mortality with limited attributes to expedite patient prognosis and triage: retrospective observational study</article-title>
          <source>JMIRx Med</source>
          <year>2021</year>
          <volume>2</volume>
          <issue>4</issue>
          <fpage>e29392</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://med.jmirx.org/2021/4/e29392/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/29392</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Moquillaza Alcántara</surname>
              <given-names>VH</given-names>
            </name>
          </person-group>
          <article-title>Peer review of "Machine learning–based prediction of COVID-19 mortality with limited attributes to expedite patient prognosis and triage: retrospective observational study"</article-title>
          <source>JMIRx Med</source>
          <year>2021</year>
          <volume>2</volume>
          <issue>4</issue>
          <fpage>e34083</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://med.jmirx.org/2021/4/e34083/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/34083</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
