The following bash script downloads the summaries into an XML file.
#!/usr/bin/env bash
# This requires the ncbi tool `gene2xml`
# sudo apt install ncbi-tools-bin
DIR="download_cache"
mkdir -p "$DIR"
URL="ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/ASN_BINARY/Mammalia/Mus_musculus.ags.gz"
wget -S -O - "$URL" | gzip -d | gene2xml -b T -l T | gzip -9 > "$DIR/mm.xml.gz"
URL="ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/ASN_BINARY/Mammalia/Homo_sapiens.ags.gz"
wget -S -O - "$URL" | gzip -d | gene2xml -b T -l T | gzip -9 > "$DIR/hs.xml.gz"
These XML files can be converted to tables along these lines, viz.:
import xml.etree.ElementTree as ET
import gzip
from pathlib import Path
from tcga.utils import mkdir, unlist1, relpath
def parse(src):
def find_text(e, path):
e = e.findall(path)
if e:
return unlist1(e).text
else:
return ""
yield ('gene_id', 'description', 'summary')
with gzip.open(src, mode='r') as fd:
e: ET.Element
for (x, e) in ET.iterparse(fd, events=['end']):
if e.tag == "Entrezgene":
gene_id = find_text(e, "**/Gene-track_geneid")
descptn = find_text(e, "**/Gene-ref_desc")
summary = find_text(e, "Entrezgene_summary")
yield (gene_id, descptn, summary)
e.clear()
def main():
for src in Path(__file__).parent.glob("download_cache/*.xml.gz"):
trg = mkdir(Path(__file__).with_suffix('')) / Path(src.stem).with_suffix('.tsv.gz')
if trg.is_file():
print("Skipping existing file:", relpath(trg))
else:
print("Writing:", relpath(trg))
with gzip.open(trg, mode='wt') as fd:
for t in parse(src):
print(*t, sep='\t', file=fd)
if __name__ == '__main__':
main()
Btw., a table of vertebrate gene homology is available here (it's just a tab-separated table), and can be used to impute summaries by homology.
purrr::insistently
. No need for a loop, this can be vectorised. I also recommend against a fixed-length sleep; using an exponential backoff is more polite. $\endgroup$