From 6f12bc7301d5b945e28b7efd14470cc6222973dc Mon Sep 17 00:00:00 2001 From: Antonio Hidalgo Date: Thu, 17 Jan 2019 20:25:45 +0530 Subject: [PATCH 01/28] Add the dastcom5 module from poliastro --- astroquery/dastcom5/__init__.py | 13 + astroquery/dastcom5/dastcom5.py | 591 ++++++++++++++++++++++++++++++++ 2 files changed, 604 insertions(+) create mode 100644 astroquery/dastcom5/__init__.py create mode 100644 astroquery/dastcom5/dastcom5.py diff --git a/astroquery/dastcom5/__init__.py b/astroquery/dastcom5/__init__.py new file mode 100644 index 0000000000..59c3d87ddc --- /dev/null +++ b/astroquery/dastcom5/__init__.py @@ -0,0 +1,13 @@ +"""Code related to NEOs. + +Functions related to NEOs and different NASA APIs. +All of them are coded as part of SOCIS 2017 proposal. + +Notes +----- + +The orbits returned by the functions in this package are in the +:py:class:`~poliastro.frames.HeliocentricEclipticJ2000` frame. + +""" +from poliastro.neos import neows, dastcom5 # flake8: noqa diff --git a/astroquery/dastcom5/dastcom5.py b/astroquery/dastcom5/dastcom5.py new file mode 100644 index 0000000000..6baa7ee394 --- /dev/null +++ b/astroquery/dastcom5/dastcom5.py @@ -0,0 +1,591 @@ +"""NEOs orbit from DASTCOM5 database. + +""" +import os +import re +import urllib.request +import zipfile + +import astropy.units as u +import numpy as np +import pandas as pd +from astropy.time import Time + +from poliastro.bodies import Sun +from poliastro.frames import HeliocentricEclipticJ2000 +from poliastro.twobody.angles import M_to_nu +from poliastro.twobody.orbit import Orbit + +AST_DTYPE = np.dtype( + [ + ("NO", np.int32), + ("NOBS", np.int32), + ("OBSFRST", np.int32), + ("OBSLAST", np.int32), + ("EPOCH", np.float64), + ("CALEPO", np.float64), + ("MA", np.float64), + ("W", np.float64), + ("OM", np.float64), + ("IN", np.float64), + ("EC", np.float64), + ("A", np.float64), + ("QR", np.float64), + ("TP", np.float64), + ("TPCAL", np.float64), + ("TPFRAC", np.float64), + ("SOLDAT", np.float64), + ("SRC1", np.float64), + ("SRC2", np.float64), + ("SRC3", np.float64), + ("SRC4", np.float64), + ("SRC5", np.float64), + ("SRC6", np.float64), + ("SRC7", np.float64), + ("SRC8", np.float64), + ("SRC9", np.float64), + ("SRC10", np.float64), + ("SRC11", np.float64), + ("SRC12", np.float64), + ("SRC13", np.float64), + ("SRC14", np.float64), + ("SRC15", np.float64), + ("SRC16", np.float64), + ("SRC17", np.float64), + ("SRC18", np.float64), + ("SRC19", np.float64), + ("SRC20", np.float64), + ("SRC21", np.float64), + ("SRC22", np.float64), + ("SRC23", np.float64), + ("SRC24", np.float64), + ("SRC25", np.float64), + ("SRC26", np.float64), + ("SRC27", np.float64), + ("SRC28", np.float64), + ("SRC29", np.float64), + ("SRC30", np.float64), + ("SRC31", np.float64), + ("SRC32", np.float64), + ("SRC33", np.float64), + ("SRC34", np.float64), + ("SRC35", np.float64), + ("SRC36", np.float64), + ("SRC37", np.float64), + ("SRC38", np.float64), + ("SRC39", np.float64), + ("SRC40", np.float64), + ("SRC41", np.float64), + ("SRC42", np.float64), + ("SRC43", np.float64), + ("SRC44", np.float64), + ("SRC45", np.float64), + ("PRELTV", np.int8), + ("SPHMX3", np.int8), + ("SPHMX5", np.int8), + ("JGSEP", np.int8), + ("TWOBOD", np.int8), + ("NSATS", np.int8), + ("UPARM", np.int8), + ("LSRC", np.int8), + ("NDEL", np.int16), + ("NDOP", np.int16), + ("H", np.float32), + ("G", np.float32), + ("A1", np.float32), + ("A2", np.float32), + ("A3", np.float32), + ("R0", np.float32), + ("ALN", np.float32), + ("NM", np.float32), + ("NN", np.float32), + ("NK", np.float32), + ("LGK", np.float32), + ("RHO", np.float32), + ("AMRAT", np.float32), + ("ALF", np.float32), + ("DEL", np.float32), + ("SPHLM3", np.float32), + ("SPHLM5", np.float32), + ("RP", np.float32), + ("GM", np.float32), + ("RAD", np.float32), + ("EXTNT1", np.float32), + ("EXTNT2", np.float32), + ("EXTNT3", np.float32), + ("MOID", np.float32), + ("ALBEDO", np.float32), + ("BVCI", np.float32), + ("UBCI", np.float32), + ("IRCI", np.float32), + ("RMSW", np.float32), + ("RMSU", np.float32), + ("RMSN", np.float32), + ("RMSNT", np.float32), + ("RMSH", np.float32), + ("EQUNOX", "|S4"), + ("PENAM", "|S6"), + ("SBNAM", "|S12"), + ("SPTYPT", "|S5"), + ("SPTYPS", "|S5"), + ("DARC", "|S9"), + ("COMNT1", "|S41"), + ("COMNT2", "|S80"), + ("DESIG", "|S13"), + ("ASTEST", "|S8"), + ("IREF", "|S10"), + ("ASTNAM", "|S18"), + ] +) + +COM_DTYPE = np.dtype( + [ + ("NO", np.int32), + ("NOBS", np.int32), + ("OBSFRST", np.int32), + ("OBSLAST", np.int32), + ("EPOCH", np.float64), + ("CALEPO", np.float64), + ("MA", np.float64), + ("W", np.float64), + ("OM", np.float64), + ("IN", np.float64), + ("EC", np.float64), + ("A", np.float64), + ("QR", np.float64), + ("TP", np.float64), + ("TPCAL", np.float64), + ("TPFRAC", np.float64), + ("SOLDAT", np.float64), + ("SRC1", np.float64), + ("SRC2", np.float64), + ("SRC3", np.float64), + ("SRC4", np.float64), + ("SRC5", np.float64), + ("SRC6", np.float64), + ("SRC7", np.float64), + ("SRC8", np.float64), + ("SRC9", np.float64), + ("SRC10", np.float64), + ("SRC11", np.float64), + ("SRC12", np.float64), + ("SRC13", np.float64), + ("SRC14", np.float64), + ("SRC15", np.float64), + ("SRC16", np.float64), + ("SRC17", np.float64), + ("SRC18", np.float64), + ("SRC19", np.float64), + ("SRC20", np.float64), + ("SRC21", np.float64), + ("SRC22", np.float64), + ("SRC23", np.float64), + ("SRC24", np.float64), + ("SRC25", np.float64), + ("SRC26", np.float64), + ("SRC27", np.float64), + ("SRC28", np.float64), + ("SRC29", np.float64), + ("SRC30", np.float64), + ("SRC31", np.float64), + ("SRC32", np.float64), + ("SRC33", np.float64), + ("SRC34", np.float64), + ("SRC35", np.float64), + ("SRC36", np.float64), + ("SRC37", np.float64), + ("SRC38", np.float64), + ("SRC39", np.float64), + ("SRC40", np.float64), + ("SRC41", np.float64), + ("SRC42", np.float64), + ("SRC43", np.float64), + ("SRC44", np.float64), + ("SRC45", np.float64), + ("SRC46", np.float64), + ("SRC47", np.float64), + ("SRC48", np.float64), + ("SRC49", np.float64), + ("SRC50", np.float64), + ("SRC51", np.float64), + ("SRC52", np.float64), + ("SRC53", np.float64), + ("SRC54", np.float64), + ("SRC55", np.float64), + ("PRELTV", np.int8), + ("SPHMX3", np.int8), + ("SPHMX5", np.int8), + ("JGSEP", np.int8), + ("TWOBOD", np.int8), + ("NSATS", np.int8), + ("UPARM", np.int8), + ("LSRC", np.int8), + ("IPYR", np.int16), + ("NDEL", np.int16), + ("NDOP", np.int16), + ("NOBSMT", np.int16), + ("NOBSMN", np.int16), + ("H", np.float32), + ("G", np.float32), + ("M1 (MT)", np.float32), + ("M2 (MN)", np.float32), + ("K1 (MTSMT)", np.float32), + ("K2 (MNSMT)", np.float32), + ("PHCOF (MNP)", np.float32), + ("A1", np.float32), + ("A2", np.float32), + ("A3", np.float32), + ("DT", np.float32), + ("R0", np.float32), + ("ALN", np.float32), + ("NM", np.float32), + ("NN", np.float32), + ("NK", np.float32), + ("S0", np.float32), + ("TCL", np.float32), + ("RHO", np.float32), + ("AMRAT", np.float32), + ("AJ1", np.float32), + ("AJ2", np.float32), + ("ET1", np.float32), + ("ET2", np.float32), + ("DTH", np.float32), + ("ALF", np.float32), + ("DEL", np.float32), + ("SPHLM3", np.float32), + ("SPHLM5", np.float32), + ("RP", np.float32), + ("GM", np.float32), + ("RAD", np.float32), + ("EXTNT1", np.float32), + ("EXTNT2", np.float32), + ("EXTNT3", np.float32), + ("MOID", np.float32), + ("ALBEDO", np.float32), + ("RMSW", np.float32), + ("RMSU", np.float32), + ("RMSN", np.float32), + ("RMSNT", np.float32), + ("RMSMT", np.float32), + ("RMSMN", np.float32), + ("EQUNOX", "|S4"), + ("PENAM", "|S6"), + ("SBNAM", "|S12"), + ("DARC", "|S9"), + ("COMNT3", "|S49"), + ("COMNT2", "|S80"), + ("DESIG", "|S13"), + ("COMEST", "|S14"), + ("IREF", "|S10"), + ("COMNAM", "|S29"), + ] +) + +POLIASTRO_LOCAL_PATH = os.path.join(os.path.expanduser("~"), ".poliastro") +DBS_LOCAL_PATH = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5", "dat") +AST_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") +COM_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") + +FTP_DB_URL = "ftp://ssd.jpl.nasa.gov/pub/ssd/" + + +def asteroid_db(): + """Return complete DASTCOM5 asteroid database. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + with open(AST_DB_PATH, "rb") as f: + f.seek(835, os.SEEK_SET) + data = np.fromfile(f, dtype=AST_DTYPE) + return data + + + +def comet_db(): + """Return complete DASTCOM5 comet database. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + with open(COM_DB_PATH, "rb") as f: + f.seek(976, os.SEEK_SET) + data = np.fromfile(f, dtype=COM_DTYPE) + return data + + +def orbit_from_name(name): + """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a name. + + Retrieve info from JPL DASTCOM5 database. + + Parameters + ---------- + name : str + NEO name. + + Returns + ------- + orbit : list (~poliastro.twobody.orbit.Orbit) + NEO orbits. + + """ + records = record_from_name(name) + orbits = [] + for record in records: + orbits.append(orbit_from_record(record)) + return orbits + + +def orbit_from_record(record): + """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a record. + + Retrieve info from JPL DASTCOM5 database. + + Parameters + ---------- + record : int + Object record. + + Returns + ------- + orbit : ~poliastro.twobody.orbit.Orbit + NEO orbit. + + """ + body_data = read_record(record) + a = body_data["A"].item() * u.au + ecc = body_data["EC"].item() * u.one + inc = body_data["IN"].item() * u.deg + raan = body_data["OM"].item() * u.deg + argp = body_data["W"].item() * u.deg + m = body_data["MA"].item() * u.deg + nu = M_to_nu(m, ecc) + epoch = Time(body_data["EPOCH"].item(), format="jd", scale="tdb") + + orbit = Orbit.from_classical(Sun, a, ecc, inc, raan, argp, nu, epoch) + orbit._frame = HeliocentricEclipticJ2000(obstime=epoch) + return orbit + + +def record_from_name(name): + """Search `dastcom.idx` and return logical records that match a given string. + + Body name, SPK-ID, or alternative designations can be used. + + Parameters + ---------- + name : str + Body name. + + Returns + ------- + records : list (int) + DASTCOM5 database logical records matching str. + + """ + records = [] + lines = string_record_from_name(name) + for line in lines: + records.append(int(line[:6].lstrip())) + return records + + +def string_record_from_name(name): + """Search `dastcom.idx` and return body full record. + + Search DASTCOM5 index and return body records that match string, + containing logical record, name, alternative designations, SPK-ID, etc. + + Parameters + ---------- + name : str + Body name. + + Returns + ------- + lines: list(str) + Body records + """ + + idx_path = os.path.join(DBS_LOCAL_PATH, "dastcom.idx") + lines = [] + with open(idx_path, "r") as inF: + for line in inF: + if re.search(r"\b" + name.casefold() + r"\b", line.casefold()): + lines.append(line) + return lines + + +def read_headers(): + """Read `DASTCOM5` headers and return asteroid and comet headers. + + Headers are two numpy arrays with custom dtype. + + Returns + ------- + ast_header, com_header : tuple (numpy.ndarray) + DASTCOM5 headers. + + """ + + ast_path = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") + ast_dtype = np.dtype( + [ + ("IBIAS1", np.int32), + ("BEGINP1", "|S8"), + ("BEGINP2", "|S8"), + ("BEGINP3", "|S8"), + ("ENDPT1", "|S8"), + ("ENDPT2", "|S8"), + ("ENDPT3", "|S8"), + ("CALDATE", "|S19"), + ("JDDATE", np.float64), + ("FTYP", "|S1"), + ("BYTE2A", np.int16), + ("IBIAS0", np.int32), + ] + ) + + with open(ast_path, "rb") as f: + ast_header = np.fromfile(f, dtype=ast_dtype, count=1) + + com_path = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") + com_dtype = np.dtype( + [ + ("IBIAS2", np.int32), + ("BEGINP1", "|S8"), + ("BEGINP2", "|S8"), + ("BEGINP3", "|S8"), + ("ENDPT1", "|S8"), + ("ENDPT2", "|S8"), + ("ENDPT3", "|S8"), + ("CALDATE", "|S19"), + ("JDDATE", np.float64), + ("FTYP", "|S1"), + ("BYTE2C", np.int16), + ] + ) + + with open(com_path, "rb") as f: + com_header = np.fromfile(f, dtype=com_dtype, count=1) + + return ast_header, com_header + + +def read_record(record): + """Read `DASTCOM5` record and return body data. + + Body data consists of numpy array with custom dtype. + + Parameters + ---------- + record : int + Body record. + + Returns + ------- + body_data : numpy.ndarray + Body information. + + """ + ast_header, com_header = read_headers() + ast_path = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") + com_path = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") + # ENDPT1 indicates end of numbered asteroids records + if record <= int(ast_header["ENDPT2"][0].item()): + # ENDPT2 indicates end of unnumbered asteroids records + if record <= int(ast_header["ENDPT1"][0].item()): + # phis_rec = record_size * (record_number - IBIAS - 1 (header record)) + phis_rec = 835 * (record - ast_header["IBIAS0"][0].item() - 1) + else: + phis_rec = 835 * (record - ast_header["IBIAS1"][0].item() - 1) + + with open(ast_path, "rb") as f: + f.seek(phis_rec, os.SEEK_SET) + body_data = np.fromfile(f, dtype=AST_DTYPE, count=1) + else: + phis_rec = 976 * (record - com_header["IBIAS2"][0].item() - 1) + with open(com_path, "rb") as f: + f.seek(phis_rec, os.SEEK_SET) + body_data = np.fromfile(f, dtype=COM_DTYPE, count=1) + return body_data + + +def download_dastcom5(): + """Downloads DASTCOM5 database. + + Downloads and unzip DASTCOM5 file in default poliastro path (~/.poliastro). + + """ + + dastcom5_dir = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5") + dastcom5_zip_path = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5.zip") + + if os.path.isdir(dastcom5_dir): + raise FileExistsError( + "dastcom5 is already created in " + os.path.abspath(dastcom5_dir) + ) + if not zipfile.is_zipfile(dastcom5_zip_path): + if not os.path.isdir(POLIASTRO_LOCAL_PATH): + os.makedirs(POLIASTRO_LOCAL_PATH) + + urllib.request.urlretrieve( + FTP_DB_URL + "dastcom5.zip", dastcom5_zip_path, _show_download_progress + ) + with zipfile.ZipFile(dastcom5_zip_path) as myzip: + myzip.extractall(POLIASTRO_LOCAL_PATH) + + +def _show_download_progress(transferred, block, totalsize): + trans_mb = transferred * block / (1024 * 1024) + total_mb = totalsize / (1024 * 1024) + print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) + + +def entire_db(): + """Return complete DASTCOM5 database. + + Merge asteroid and comet databases, only with fields + related to orbital data, discarding the rest. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + ast_database = asteroid_db() + com_database = comet_db() + + ast_database = pd.DataFrame( + ast_database[ + list(ast_database.dtype.names[:17]) + + list(ast_database.dtype.names[-4:-3]) + + list(ast_database.dtype.names[-2:]) + ] + ) + ast_database.rename( + columns={"ASTNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True + ) + com_database = pd.DataFrame( + com_database[ + list(com_database.dtype.names[:17]) + + list(com_database.dtype.names[-4:-3]) + + list(com_database.dtype.names[-2:]) + ] + ) + com_database.rename( + columns={"COMNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True + ) + df = ast_database.append(com_database, ignore_index=True) + df[["NAME", "DESIG", "IREF"]] = df[["NAME", "DESIG", "IREF"]].apply( + lambda x: x.str.strip().str.decode("utf-8") + ) + return df From 85e6ca07084cf43a78c8e8428602339d796bbe6d Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 17 Jan 2019 20:27:00 +0530 Subject: [PATCH 02/28] Add the init file --- astroquery/dastcom5/__init__.py | 298 +++++++++++++++++++++++++++++++- 1 file changed, 289 insertions(+), 9 deletions(-) mode change 100644 => 100755 astroquery/dastcom5/__init__.py diff --git a/astroquery/dastcom5/__init__.py b/astroquery/dastcom5/__init__.py old mode 100644 new mode 100755 index 59c3d87ddc..105b4e52fe --- a/astroquery/dastcom5/__init__.py +++ b/astroquery/dastcom5/__init__.py @@ -1,13 +1,293 @@ -"""Code related to NEOs. - -Functions related to NEOs and different NASA APIs. -All of them are coded as part of SOCIS 2017 proposal. +""" +DASTCOM5 Query Tool +=================== -Notes ------ +:Authors: +1) Antonio Hidalgo (antoniohidalgo.inves@gmail.com) +2) Juan Luis Cano Rodríguez (juanlu001@gmail.com) -The orbits returned by the functions in this package are in the -:py:class:`~poliastro.frames.HeliocentricEclipticJ2000` frame. +This module contains various methods for querying the +DASTCOM5. +All of the methods are coded as part of SOCIS 2017 for poliastro by Antonio Hidalgo[1]. """ -from poliastro.neos import neows, dastcom5 # flake8: noqa +from astropy import config as _config + + +class Conf(_config.ConfigNamespace): + AST_DTYPE = np.dtype( + [ + ("NO", np.int32), + ("NOBS", np.int32), + ("OBSFRST", np.int32), + ("OBSLAST", np.int32), + ("EPOCH", np.float64), + ("CALEPO", np.float64), + ("MA", np.float64), + ("W", np.float64), + ("OM", np.float64), + ("IN", np.float64), + ("EC", np.float64), + ("A", np.float64), + ("QR", np.float64), + ("TP", np.float64), + ("TPCAL", np.float64), + ("TPFRAC", np.float64), + ("SOLDAT", np.float64), + ("SRC1", np.float64), + ("SRC2", np.float64), + ("SRC3", np.float64), + ("SRC4", np.float64), + ("SRC5", np.float64), + ("SRC6", np.float64), + ("SRC7", np.float64), + ("SRC8", np.float64), + ("SRC9", np.float64), + ("SRC10", np.float64), + ("SRC11", np.float64), + ("SRC12", np.float64), + ("SRC13", np.float64), + ("SRC14", np.float64), + ("SRC15", np.float64), + ("SRC16", np.float64), + ("SRC17", np.float64), + ("SRC18", np.float64), + ("SRC19", np.float64), + ("SRC20", np.float64), + ("SRC21", np.float64), + ("SRC22", np.float64), + ("SRC23", np.float64), + ("SRC24", np.float64), + ("SRC25", np.float64), + ("SRC26", np.float64), + ("SRC27", np.float64), + ("SRC28", np.float64), + ("SRC29", np.float64), + ("SRC30", np.float64), + ("SRC31", np.float64), + ("SRC32", np.float64), + ("SRC33", np.float64), + ("SRC34", np.float64), + ("SRC35", np.float64), + ("SRC36", np.float64), + ("SRC37", np.float64), + ("SRC38", np.float64), + ("SRC39", np.float64), + ("SRC40", np.float64), + ("SRC41", np.float64), + ("SRC42", np.float64), + ("SRC43", np.float64), + ("SRC44", np.float64), + ("SRC45", np.float64), + ("PRELTV", np.int8), + ("SPHMX3", np.int8), + ("SPHMX5", np.int8), + ("JGSEP", np.int8), + ("TWOBOD", np.int8), + ("NSATS", np.int8), + ("UPARM", np.int8), + ("LSRC", np.int8), + ("NDEL", np.int16), + ("NDOP", np.int16), + ("H", np.float32), + ("G", np.float32), + ("A1", np.float32), + ("A2", np.float32), + ("A3", np.float32), + ("R0", np.float32), + ("ALN", np.float32), + ("NM", np.float32), + ("NN", np.float32), + ("NK", np.float32), + ("LGK", np.float32), + ("RHO", np.float32), + ("AMRAT", np.float32), + ("ALF", np.float32), + ("DEL", np.float32), + ("SPHLM3", np.float32), + ("SPHLM5", np.float32), + ("RP", np.float32), + ("GM", np.float32), + ("RAD", np.float32), + ("EXTNT1", np.float32), + ("EXTNT2", np.float32), + ("EXTNT3", np.float32), + ("MOID", np.float32), + ("ALBEDO", np.float32), + ("BVCI", np.float32), + ("UBCI", np.float32), + ("IRCI", np.float32), + ("RMSW", np.float32), + ("RMSU", np.float32), + ("RMSN", np.float32), + ("RMSNT", np.float32), + ("RMSH", np.float32), + ("EQUNOX", "|S4"), + ("PENAM", "|S6"), + ("SBNAM", "|S12"), + ("SPTYPT", "|S5"), + ("SPTYPS", "|S5"), + ("DARC", "|S9"), + ("COMNT1", "|S41"), + ("COMNT2", "|S80"), + ("DESIG", "|S13"), + ("ASTEST", "|S8"), + ("IREF", "|S10"), + ("ASTNAM", "|S18"), + ] + ) + + COM_DTYPE = np.dtype( + [ + ("NO", np.int32), + ("NOBS", np.int32), + ("OBSFRST", np.int32), + ("OBSLAST", np.int32), + ("EPOCH", np.float64), + ("CALEPO", np.float64), + ("MA", np.float64), + ("W", np.float64), + ("OM", np.float64), + ("IN", np.float64), + ("EC", np.float64), + ("A", np.float64), + ("QR", np.float64), + ("TP", np.float64), + ("TPCAL", np.float64), + ("TPFRAC", np.float64), + ("SOLDAT", np.float64), + ("SRC1", np.float64), + ("SRC2", np.float64), + ("SRC3", np.float64), + ("SRC4", np.float64), + ("SRC5", np.float64), + ("SRC6", np.float64), + ("SRC7", np.float64), + ("SRC8", np.float64), + ("SRC9", np.float64), + ("SRC10", np.float64), + ("SRC11", np.float64), + ("SRC12", np.float64), + ("SRC13", np.float64), + ("SRC14", np.float64), + ("SRC15", np.float64), + ("SRC16", np.float64), + ("SRC17", np.float64), + ("SRC18", np.float64), + ("SRC19", np.float64), + ("SRC20", np.float64), + ("SRC21", np.float64), + ("SRC22", np.float64), + ("SRC23", np.float64), + ("SRC24", np.float64), + ("SRC25", np.float64), + ("SRC26", np.float64), + ("SRC27", np.float64), + ("SRC28", np.float64), + ("SRC29", np.float64), + ("SRC30", np.float64), + ("SRC31", np.float64), + ("SRC32", np.float64), + ("SRC33", np.float64), + ("SRC34", np.float64), + ("SRC35", np.float64), + ("SRC36", np.float64), + ("SRC37", np.float64), + ("SRC38", np.float64), + ("SRC39", np.float64), + ("SRC40", np.float64), + ("SRC41", np.float64), + ("SRC42", np.float64), + ("SRC43", np.float64), + ("SRC44", np.float64), + ("SRC45", np.float64), + ("SRC46", np.float64), + ("SRC47", np.float64), + ("SRC48", np.float64), + ("SRC49", np.float64), + ("SRC50", np.float64), + ("SRC51", np.float64), + ("SRC52", np.float64), + ("SRC53", np.float64), + ("SRC54", np.float64), + ("SRC55", np.float64), + ("PRELTV", np.int8), + ("SPHMX3", np.int8), + ("SPHMX5", np.int8), + ("JGSEP", np.int8), + ("TWOBOD", np.int8), + ("NSATS", np.int8), + ("UPARM", np.int8), + ("LSRC", np.int8), + ("IPYR", np.int16), + ("NDEL", np.int16), + ("NDOP", np.int16), + ("NOBSMT", np.int16), + ("NOBSMN", np.int16), + ("H", np.float32), + ("G", np.float32), + ("M1 (MT)", np.float32), + ("M2 (MN)", np.float32), + ("K1 (MTSMT)", np.float32), + ("K2 (MNSMT)", np.float32), + ("PHCOF (MNP)", np.float32), + ("A1", np.float32), + ("A2", np.float32), + ("A3", np.float32), + ("DT", np.float32), + ("R0", np.float32), + ("ALN", np.float32), + ("NM", np.float32), + ("NN", np.float32), + ("NK", np.float32), + ("S0", np.float32), + ("TCL", np.float32), + ("RHO", np.float32), + ("AMRAT", np.float32), + ("AJ1", np.float32), + ("AJ2", np.float32), + ("ET1", np.float32), + ("ET2", np.float32), + ("DTH", np.float32), + ("ALF", np.float32), + ("DEL", np.float32), + ("SPHLM3", np.float32), + ("SPHLM5", np.float32), + ("RP", np.float32), + ("GM", np.float32), + ("RAD", np.float32), + ("EXTNT1", np.float32), + ("EXTNT2", np.float32), + ("EXTNT3", np.float32), + ("MOID", np.float32), + ("ALBEDO", np.float32), + ("RMSW", np.float32), + ("RMSU", np.float32), + ("RMSN", np.float32), + ("RMSNT", np.float32), + ("RMSMT", np.float32), + ("RMSMN", np.float32), + ("EQUNOX", "|S4"), + ("PENAM", "|S6"), + ("SBNAM", "|S12"), + ("DARC", "|S9"), + ("COMNT3", "|S49"), + ("COMNT2", "|S80"), + ("DESIG", "|S13"), + ("COMEST", "|S14"), + ("IREF", "|S10"), + ("COMNAM", "|S29"), + ] + ) + ASTROQUERY_LOCAL_PATH = os.path.join(os.path.expanduser("~"), ".astroquery") + DBS_LOCAL_PATH = os.path.join(ASTROQUERY_LOCAL_PATH, "dastcom5", "dat") + AST_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") + COM_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") + FTP_DB_URL = "ftp://ssd.jpl.nasa.gov/pub/ssd/" + + +conf = Conf() + +from .core import * + +__all__ = [Dastcom5, Dastcom5Class] From 8fd6b0e3d523d96d7266f3810548d3913a298653 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 17 Jan 2019 21:02:40 +0530 Subject: [PATCH 03/28] Add the core package for querying DASTCOM5 --- astroquery/dastcom5/core.py | 188 ++++++++++ astroquery/dastcom5/dastcom5.py | 591 -------------------------------- 2 files changed, 188 insertions(+), 591 deletions(-) create mode 100755 astroquery/dastcom5/core.py delete mode 100644 astroquery/dastcom5/dastcom5.py diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py new file mode 100755 index 0000000000..38786c4f9a --- /dev/null +++ b/astroquery/dastcom5/core.py @@ -0,0 +1,188 @@ +import re +import os +import urllib.request +import zipfile + +from astropy.time import Time +from astropy.table import Table + +from . import conf +from ..query import BaseQuery +from ..utils import async_to_sync + +@async_to_sync +class Dastcom5Class(BaseQuery): + """ + Class for querying NEOs orbit from DASTCOM5. + """ + + def __init__(self, spk_id=None, name=None): + """ + Parameters: + ---------- + api_key : str + NASA OPEN APIs key (default: `DEMO_KEY`) + """ + super(NeowsClass, self).__init__() + self.spk_id = spk_id + self.name = name + self.local_path = conf.ASTROQUERY_LOCAL_PATH + self.ast_path = conf.AST_DB_PATH + self.com_path = conf.COM_DB_PATH + self.com_dtype = conf.COM_DTYPE + self.ast_dtype = conf.AST_DB_PATH + + + def download_dastcom5(self): + """Downloads DASTCOM5 database. + + Downloads and unzip DASTCOM5 file in default poliastro path (~/.poliastro). + + """ + + dastcom5_dir = os.path.join(self.local_path, "dastcom5") + dastcom5_zip_path = os.path.join(self.local_path, "dastcom5.zip") + + if os.path.isdir(dastcom5_dir): + raise FileExistsError( + "dastcom5 is already created in " + os.path.abspath(dastcom5_dir) + ) + if not zipfile.is_zipfile(dastcom5_zip_path): + if not os.path.isdir(self.local_path): + os.makedirs(self.local_path) + + urllib.request.urlretrieve( + FTP_DB_URL + "dastcom5.zip", dastcom5_zip_path, _show_download_progress + ) + with zipfile.ZipFile(dastcom5_zip_path) as myzip: + myzip.extractall(self.local_path) + + + def asteroid_db(): + """Return complete DASTCOM5 asteroid database. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + with open(AST_DB_PATH, "rb") as f: + f.seek(835, os.SEEK_SET) + data = np.fromfile(f, dtype=AST_DTYPE) + return data + + + + def comet_db(): + """Return complete DASTCOM5 comet database. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + with open(COM_DB_PATH, "rb") as f: + f.seek(976, os.SEEK_SET) + data = np.fromfile(f, dtype=COM_DTYPE) + return data + + + def _spk_id_from_name(self, name, cache=True): + payload = { + "sstr": name, + "orb": "0", + "log": "0", + "old": "0", + "cov": "0", + "cad": "0", + } + SBDB_URL = conf.sbdb_server + response = self._request( + "GET", SBDB_URL, params=payload, timeout=self.TIMEOUT, cache=cahe) + + response.raise_for_status() + soup = BeautifulSoup(response.text, "html.parser") + + # page_identifier is used to check what type of response page we are working with. + page_identifier = soup.find(attrs={"name": "top"}) + + # If there is a 'table' sibling, the object was found. + if page_identifier.find_next_sibling("table") is not None: + data = page_identifier.find_next_sibling( + "table").table.find_all("td") + + complete_string = "" + for string in data[1].stripped_strings: + complete_string += string + " " + match = re.compile(r"Classification: ([\S\s]+) SPK-ID: (\d+)").match( + complete_string + ) + if match: + self.spk_id = match.group(2) + return match.group(2) + + # If there is a 'center' sibling, it is a page with a list of possible objects + elif page_identifier.find_next_sibling("center") is not None: + object_list = page_identifier.find_next_sibling("center").table.find_all( + "td" + ) + bodies = "" + obj_num = min(len(object_list), 3) + for body in object_list[:obj_num]: + bodies += body.string + "\n" + raise ValueError( + str(len(object_list)) + " different bodies found:\n" + bodies + ) + + # If everything else failed + raise ValueError("Object could not be found. You can visit: " + + SBDB_URL + "?sstr=" + name + " for more information.") + + def from_spk_id(self, spk_id, cache=True): + """Return :py:class:`~astropy.table.Table` given a SPK-ID. + + Retrieve info from NASA NeoWS API, and therefore + it only works with NEAs (Near Earth Asteroids). + + Parameters + ---------- + spk_id : str + SPK-ID number, which is given to each body by JPL. + + Returns + ------- + Table : ~astropy.table.Table + NEA orbit parameters. + + """ + payload = {"api_key": self.api_key or DEFAULT_API_KEY} + + NEOWS_URL = conf.neows_server + response = self._request( + "GET", NEOWS_URL + spk_id, params=payload, timeout=self.TIMEOUT, cache=cache + ) + response.raise_for_status() + + orbital_data = response.json()["orbital_data"] + self.name = response.json()["name"] + abs_magnitude = response.json()["absolute_magnitude_h"] + column1 = ["name", "absolute_magnitude_h"] + column2 = [self.name, abs_magnitude] + column1.extend(orbital_data.keys()) + column2.extend(orbital_data.values()) + data = Table([column1, column2], names=("Parameters", "Values")) + self.epoch = Time( + float(orbital_data["epoch_osculation"]), format="jd", scale="tdb" + ) + + return data + + def from_name(self, name, cache=True): + if not self.spk_id: + self._spk_id_from_name(name=name, cache=cache) + return self.from_spk_id(spk_id=self.spk_id, cache=cache) + + +Neows = NeowsClass() diff --git a/astroquery/dastcom5/dastcom5.py b/astroquery/dastcom5/dastcom5.py deleted file mode 100644 index 6baa7ee394..0000000000 --- a/astroquery/dastcom5/dastcom5.py +++ /dev/null @@ -1,591 +0,0 @@ -"""NEOs orbit from DASTCOM5 database. - -""" -import os -import re -import urllib.request -import zipfile - -import astropy.units as u -import numpy as np -import pandas as pd -from astropy.time import Time - -from poliastro.bodies import Sun -from poliastro.frames import HeliocentricEclipticJ2000 -from poliastro.twobody.angles import M_to_nu -from poliastro.twobody.orbit import Orbit - -AST_DTYPE = np.dtype( - [ - ("NO", np.int32), - ("NOBS", np.int32), - ("OBSFRST", np.int32), - ("OBSLAST", np.int32), - ("EPOCH", np.float64), - ("CALEPO", np.float64), - ("MA", np.float64), - ("W", np.float64), - ("OM", np.float64), - ("IN", np.float64), - ("EC", np.float64), - ("A", np.float64), - ("QR", np.float64), - ("TP", np.float64), - ("TPCAL", np.float64), - ("TPFRAC", np.float64), - ("SOLDAT", np.float64), - ("SRC1", np.float64), - ("SRC2", np.float64), - ("SRC3", np.float64), - ("SRC4", np.float64), - ("SRC5", np.float64), - ("SRC6", np.float64), - ("SRC7", np.float64), - ("SRC8", np.float64), - ("SRC9", np.float64), - ("SRC10", np.float64), - ("SRC11", np.float64), - ("SRC12", np.float64), - ("SRC13", np.float64), - ("SRC14", np.float64), - ("SRC15", np.float64), - ("SRC16", np.float64), - ("SRC17", np.float64), - ("SRC18", np.float64), - ("SRC19", np.float64), - ("SRC20", np.float64), - ("SRC21", np.float64), - ("SRC22", np.float64), - ("SRC23", np.float64), - ("SRC24", np.float64), - ("SRC25", np.float64), - ("SRC26", np.float64), - ("SRC27", np.float64), - ("SRC28", np.float64), - ("SRC29", np.float64), - ("SRC30", np.float64), - ("SRC31", np.float64), - ("SRC32", np.float64), - ("SRC33", np.float64), - ("SRC34", np.float64), - ("SRC35", np.float64), - ("SRC36", np.float64), - ("SRC37", np.float64), - ("SRC38", np.float64), - ("SRC39", np.float64), - ("SRC40", np.float64), - ("SRC41", np.float64), - ("SRC42", np.float64), - ("SRC43", np.float64), - ("SRC44", np.float64), - ("SRC45", np.float64), - ("PRELTV", np.int8), - ("SPHMX3", np.int8), - ("SPHMX5", np.int8), - ("JGSEP", np.int8), - ("TWOBOD", np.int8), - ("NSATS", np.int8), - ("UPARM", np.int8), - ("LSRC", np.int8), - ("NDEL", np.int16), - ("NDOP", np.int16), - ("H", np.float32), - ("G", np.float32), - ("A1", np.float32), - ("A2", np.float32), - ("A3", np.float32), - ("R0", np.float32), - ("ALN", np.float32), - ("NM", np.float32), - ("NN", np.float32), - ("NK", np.float32), - ("LGK", np.float32), - ("RHO", np.float32), - ("AMRAT", np.float32), - ("ALF", np.float32), - ("DEL", np.float32), - ("SPHLM3", np.float32), - ("SPHLM5", np.float32), - ("RP", np.float32), - ("GM", np.float32), - ("RAD", np.float32), - ("EXTNT1", np.float32), - ("EXTNT2", np.float32), - ("EXTNT3", np.float32), - ("MOID", np.float32), - ("ALBEDO", np.float32), - ("BVCI", np.float32), - ("UBCI", np.float32), - ("IRCI", np.float32), - ("RMSW", np.float32), - ("RMSU", np.float32), - ("RMSN", np.float32), - ("RMSNT", np.float32), - ("RMSH", np.float32), - ("EQUNOX", "|S4"), - ("PENAM", "|S6"), - ("SBNAM", "|S12"), - ("SPTYPT", "|S5"), - ("SPTYPS", "|S5"), - ("DARC", "|S9"), - ("COMNT1", "|S41"), - ("COMNT2", "|S80"), - ("DESIG", "|S13"), - ("ASTEST", "|S8"), - ("IREF", "|S10"), - ("ASTNAM", "|S18"), - ] -) - -COM_DTYPE = np.dtype( - [ - ("NO", np.int32), - ("NOBS", np.int32), - ("OBSFRST", np.int32), - ("OBSLAST", np.int32), - ("EPOCH", np.float64), - ("CALEPO", np.float64), - ("MA", np.float64), - ("W", np.float64), - ("OM", np.float64), - ("IN", np.float64), - ("EC", np.float64), - ("A", np.float64), - ("QR", np.float64), - ("TP", np.float64), - ("TPCAL", np.float64), - ("TPFRAC", np.float64), - ("SOLDAT", np.float64), - ("SRC1", np.float64), - ("SRC2", np.float64), - ("SRC3", np.float64), - ("SRC4", np.float64), - ("SRC5", np.float64), - ("SRC6", np.float64), - ("SRC7", np.float64), - ("SRC8", np.float64), - ("SRC9", np.float64), - ("SRC10", np.float64), - ("SRC11", np.float64), - ("SRC12", np.float64), - ("SRC13", np.float64), - ("SRC14", np.float64), - ("SRC15", np.float64), - ("SRC16", np.float64), - ("SRC17", np.float64), - ("SRC18", np.float64), - ("SRC19", np.float64), - ("SRC20", np.float64), - ("SRC21", np.float64), - ("SRC22", np.float64), - ("SRC23", np.float64), - ("SRC24", np.float64), - ("SRC25", np.float64), - ("SRC26", np.float64), - ("SRC27", np.float64), - ("SRC28", np.float64), - ("SRC29", np.float64), - ("SRC30", np.float64), - ("SRC31", np.float64), - ("SRC32", np.float64), - ("SRC33", np.float64), - ("SRC34", np.float64), - ("SRC35", np.float64), - ("SRC36", np.float64), - ("SRC37", np.float64), - ("SRC38", np.float64), - ("SRC39", np.float64), - ("SRC40", np.float64), - ("SRC41", np.float64), - ("SRC42", np.float64), - ("SRC43", np.float64), - ("SRC44", np.float64), - ("SRC45", np.float64), - ("SRC46", np.float64), - ("SRC47", np.float64), - ("SRC48", np.float64), - ("SRC49", np.float64), - ("SRC50", np.float64), - ("SRC51", np.float64), - ("SRC52", np.float64), - ("SRC53", np.float64), - ("SRC54", np.float64), - ("SRC55", np.float64), - ("PRELTV", np.int8), - ("SPHMX3", np.int8), - ("SPHMX5", np.int8), - ("JGSEP", np.int8), - ("TWOBOD", np.int8), - ("NSATS", np.int8), - ("UPARM", np.int8), - ("LSRC", np.int8), - ("IPYR", np.int16), - ("NDEL", np.int16), - ("NDOP", np.int16), - ("NOBSMT", np.int16), - ("NOBSMN", np.int16), - ("H", np.float32), - ("G", np.float32), - ("M1 (MT)", np.float32), - ("M2 (MN)", np.float32), - ("K1 (MTSMT)", np.float32), - ("K2 (MNSMT)", np.float32), - ("PHCOF (MNP)", np.float32), - ("A1", np.float32), - ("A2", np.float32), - ("A3", np.float32), - ("DT", np.float32), - ("R0", np.float32), - ("ALN", np.float32), - ("NM", np.float32), - ("NN", np.float32), - ("NK", np.float32), - ("S0", np.float32), - ("TCL", np.float32), - ("RHO", np.float32), - ("AMRAT", np.float32), - ("AJ1", np.float32), - ("AJ2", np.float32), - ("ET1", np.float32), - ("ET2", np.float32), - ("DTH", np.float32), - ("ALF", np.float32), - ("DEL", np.float32), - ("SPHLM3", np.float32), - ("SPHLM5", np.float32), - ("RP", np.float32), - ("GM", np.float32), - ("RAD", np.float32), - ("EXTNT1", np.float32), - ("EXTNT2", np.float32), - ("EXTNT3", np.float32), - ("MOID", np.float32), - ("ALBEDO", np.float32), - ("RMSW", np.float32), - ("RMSU", np.float32), - ("RMSN", np.float32), - ("RMSNT", np.float32), - ("RMSMT", np.float32), - ("RMSMN", np.float32), - ("EQUNOX", "|S4"), - ("PENAM", "|S6"), - ("SBNAM", "|S12"), - ("DARC", "|S9"), - ("COMNT3", "|S49"), - ("COMNT2", "|S80"), - ("DESIG", "|S13"), - ("COMEST", "|S14"), - ("IREF", "|S10"), - ("COMNAM", "|S29"), - ] -) - -POLIASTRO_LOCAL_PATH = os.path.join(os.path.expanduser("~"), ".poliastro") -DBS_LOCAL_PATH = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5", "dat") -AST_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") -COM_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") - -FTP_DB_URL = "ftp://ssd.jpl.nasa.gov/pub/ssd/" - - -def asteroid_db(): - """Return complete DASTCOM5 asteroid database. - - Returns - ------- - database : numpy.ndarray - Database with custom dtype. - - """ - with open(AST_DB_PATH, "rb") as f: - f.seek(835, os.SEEK_SET) - data = np.fromfile(f, dtype=AST_DTYPE) - return data - - - -def comet_db(): - """Return complete DASTCOM5 comet database. - - Returns - ------- - database : numpy.ndarray - Database with custom dtype. - - """ - with open(COM_DB_PATH, "rb") as f: - f.seek(976, os.SEEK_SET) - data = np.fromfile(f, dtype=COM_DTYPE) - return data - - -def orbit_from_name(name): - """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a name. - - Retrieve info from JPL DASTCOM5 database. - - Parameters - ---------- - name : str - NEO name. - - Returns - ------- - orbit : list (~poliastro.twobody.orbit.Orbit) - NEO orbits. - - """ - records = record_from_name(name) - orbits = [] - for record in records: - orbits.append(orbit_from_record(record)) - return orbits - - -def orbit_from_record(record): - """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a record. - - Retrieve info from JPL DASTCOM5 database. - - Parameters - ---------- - record : int - Object record. - - Returns - ------- - orbit : ~poliastro.twobody.orbit.Orbit - NEO orbit. - - """ - body_data = read_record(record) - a = body_data["A"].item() * u.au - ecc = body_data["EC"].item() * u.one - inc = body_data["IN"].item() * u.deg - raan = body_data["OM"].item() * u.deg - argp = body_data["W"].item() * u.deg - m = body_data["MA"].item() * u.deg - nu = M_to_nu(m, ecc) - epoch = Time(body_data["EPOCH"].item(), format="jd", scale="tdb") - - orbit = Orbit.from_classical(Sun, a, ecc, inc, raan, argp, nu, epoch) - orbit._frame = HeliocentricEclipticJ2000(obstime=epoch) - return orbit - - -def record_from_name(name): - """Search `dastcom.idx` and return logical records that match a given string. - - Body name, SPK-ID, or alternative designations can be used. - - Parameters - ---------- - name : str - Body name. - - Returns - ------- - records : list (int) - DASTCOM5 database logical records matching str. - - """ - records = [] - lines = string_record_from_name(name) - for line in lines: - records.append(int(line[:6].lstrip())) - return records - - -def string_record_from_name(name): - """Search `dastcom.idx` and return body full record. - - Search DASTCOM5 index and return body records that match string, - containing logical record, name, alternative designations, SPK-ID, etc. - - Parameters - ---------- - name : str - Body name. - - Returns - ------- - lines: list(str) - Body records - """ - - idx_path = os.path.join(DBS_LOCAL_PATH, "dastcom.idx") - lines = [] - with open(idx_path, "r") as inF: - for line in inF: - if re.search(r"\b" + name.casefold() + r"\b", line.casefold()): - lines.append(line) - return lines - - -def read_headers(): - """Read `DASTCOM5` headers and return asteroid and comet headers. - - Headers are two numpy arrays with custom dtype. - - Returns - ------- - ast_header, com_header : tuple (numpy.ndarray) - DASTCOM5 headers. - - """ - - ast_path = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") - ast_dtype = np.dtype( - [ - ("IBIAS1", np.int32), - ("BEGINP1", "|S8"), - ("BEGINP2", "|S8"), - ("BEGINP3", "|S8"), - ("ENDPT1", "|S8"), - ("ENDPT2", "|S8"), - ("ENDPT3", "|S8"), - ("CALDATE", "|S19"), - ("JDDATE", np.float64), - ("FTYP", "|S1"), - ("BYTE2A", np.int16), - ("IBIAS0", np.int32), - ] - ) - - with open(ast_path, "rb") as f: - ast_header = np.fromfile(f, dtype=ast_dtype, count=1) - - com_path = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") - com_dtype = np.dtype( - [ - ("IBIAS2", np.int32), - ("BEGINP1", "|S8"), - ("BEGINP2", "|S8"), - ("BEGINP3", "|S8"), - ("ENDPT1", "|S8"), - ("ENDPT2", "|S8"), - ("ENDPT3", "|S8"), - ("CALDATE", "|S19"), - ("JDDATE", np.float64), - ("FTYP", "|S1"), - ("BYTE2C", np.int16), - ] - ) - - with open(com_path, "rb") as f: - com_header = np.fromfile(f, dtype=com_dtype, count=1) - - return ast_header, com_header - - -def read_record(record): - """Read `DASTCOM5` record and return body data. - - Body data consists of numpy array with custom dtype. - - Parameters - ---------- - record : int - Body record. - - Returns - ------- - body_data : numpy.ndarray - Body information. - - """ - ast_header, com_header = read_headers() - ast_path = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") - com_path = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") - # ENDPT1 indicates end of numbered asteroids records - if record <= int(ast_header["ENDPT2"][0].item()): - # ENDPT2 indicates end of unnumbered asteroids records - if record <= int(ast_header["ENDPT1"][0].item()): - # phis_rec = record_size * (record_number - IBIAS - 1 (header record)) - phis_rec = 835 * (record - ast_header["IBIAS0"][0].item() - 1) - else: - phis_rec = 835 * (record - ast_header["IBIAS1"][0].item() - 1) - - with open(ast_path, "rb") as f: - f.seek(phis_rec, os.SEEK_SET) - body_data = np.fromfile(f, dtype=AST_DTYPE, count=1) - else: - phis_rec = 976 * (record - com_header["IBIAS2"][0].item() - 1) - with open(com_path, "rb") as f: - f.seek(phis_rec, os.SEEK_SET) - body_data = np.fromfile(f, dtype=COM_DTYPE, count=1) - return body_data - - -def download_dastcom5(): - """Downloads DASTCOM5 database. - - Downloads and unzip DASTCOM5 file in default poliastro path (~/.poliastro). - - """ - - dastcom5_dir = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5") - dastcom5_zip_path = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5.zip") - - if os.path.isdir(dastcom5_dir): - raise FileExistsError( - "dastcom5 is already created in " + os.path.abspath(dastcom5_dir) - ) - if not zipfile.is_zipfile(dastcom5_zip_path): - if not os.path.isdir(POLIASTRO_LOCAL_PATH): - os.makedirs(POLIASTRO_LOCAL_PATH) - - urllib.request.urlretrieve( - FTP_DB_URL + "dastcom5.zip", dastcom5_zip_path, _show_download_progress - ) - with zipfile.ZipFile(dastcom5_zip_path) as myzip: - myzip.extractall(POLIASTRO_LOCAL_PATH) - - -def _show_download_progress(transferred, block, totalsize): - trans_mb = transferred * block / (1024 * 1024) - total_mb = totalsize / (1024 * 1024) - print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) - - -def entire_db(): - """Return complete DASTCOM5 database. - - Merge asteroid and comet databases, only with fields - related to orbital data, discarding the rest. - - Returns - ------- - database : numpy.ndarray - Database with custom dtype. - - """ - ast_database = asteroid_db() - com_database = comet_db() - - ast_database = pd.DataFrame( - ast_database[ - list(ast_database.dtype.names[:17]) - + list(ast_database.dtype.names[-4:-3]) - + list(ast_database.dtype.names[-2:]) - ] - ) - ast_database.rename( - columns={"ASTNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True - ) - com_database = pd.DataFrame( - com_database[ - list(com_database.dtype.names[:17]) - + list(com_database.dtype.names[-4:-3]) - + list(com_database.dtype.names[-2:]) - ] - ) - com_database.rename( - columns={"COMNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True - ) - df = ast_database.append(com_database, ignore_index=True) - df[["NAME", "DESIG", "IREF"]] = df[["NAME", "DESIG", "IREF"]].apply( - lambda x: x.str.strip().str.decode("utf-8") - ) - return df From dcf2fb5209094e8da416c9e3547c34f5f9bc821c Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 17 Jan 2019 22:16:34 +0530 Subject: [PATCH 04/28] Create a stack of tables for returning multiple orbits --- astroquery/dastcom5/__init__.py | 3 + astroquery/dastcom5/core.py | 332 +++++++++++++++++++++++--------- 2 files changed, 245 insertions(+), 90 deletions(-) diff --git a/astroquery/dastcom5/__init__.py b/astroquery/dastcom5/__init__.py index 105b4e52fe..6c66fcbbc1 100755 --- a/astroquery/dastcom5/__init__.py +++ b/astroquery/dastcom5/__init__.py @@ -11,6 +11,9 @@ All of the methods are coded as part of SOCIS 2017 for poliastro by Antonio Hidalgo[1]. """ +import os +import numpy as np + from astropy import config as _config diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 38786c4f9a..b6e90f3916 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -2,9 +2,11 @@ import os import urllib.request import zipfile +import numpy as np from astropy.time import Time -from astropy.table import Table +from astropy.table import Table, vstack +import astropy.units as u from . import conf from ..query import BaseQuery @@ -23,14 +25,16 @@ def __init__(self, spk_id=None, name=None): api_key : str NASA OPEN APIs key (default: `DEMO_KEY`) """ - super(NeowsClass, self).__init__() + super(Dastcom5Class, self).__init__() self.spk_id = spk_id self.name = name self.local_path = conf.ASTROQUERY_LOCAL_PATH + self.dbs_path = conf.DBS_LOCAL_PATH self.ast_path = conf.AST_DB_PATH self.com_path = conf.COM_DB_PATH self.com_dtype = conf.COM_DTYPE - self.ast_dtype = conf.AST_DB_PATH + self.ast_dtype = conf.AST_DTYPE + self.ftp_url = conf.FTP_DB_URL def download_dastcom5(self): @@ -52,13 +56,18 @@ def download_dastcom5(self): os.makedirs(self.local_path) urllib.request.urlretrieve( - FTP_DB_URL + "dastcom5.zip", dastcom5_zip_path, _show_download_progress + self.ftp_url + "dastcom5.zip", dastcom5_zip_path, self._show_download_progress ) with zipfile.ZipFile(dastcom5_zip_path) as myzip: myzip.extractall(self.local_path) + def _show_download_progress(self, transferred, block, totalsize): + trans_mb = transferred * block / (1024 * 1024) + total_mb = totalsize / (1024 * 1024) + print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) - def asteroid_db(): + + def asteroid_db(self): """Return complete DASTCOM5 asteroid database. Returns @@ -67,14 +76,14 @@ def asteroid_db(): Database with custom dtype. """ - with open(AST_DB_PATH, "rb") as f: + with open(self.ast_path, "rb") as f: f.seek(835, os.SEEK_SET) - data = np.fromfile(f, dtype=AST_DTYPE) + data = np.fromfile(f, dtype=self.ast_dtype) return data - def comet_db(): + def comet_db(self): """Return complete DASTCOM5 comet database. Returns @@ -83,106 +92,249 @@ def comet_db(): Database with custom dtype. """ - with open(COM_DB_PATH, "rb") as f: + with open(self.com_path, "rb") as f: f.seek(976, os.SEEK_SET) - data = np.fromfile(f, dtype=COM_DTYPE) + data = np.fromfile(f, dtype=self.com_dtype) return data - def _spk_id_from_name(self, name, cache=True): - payload = { - "sstr": name, - "orb": "0", - "log": "0", - "old": "0", - "cov": "0", - "cad": "0", - } - SBDB_URL = conf.sbdb_server - response = self._request( - "GET", SBDB_URL, params=payload, timeout=self.TIMEOUT, cache=cahe) - - response.raise_for_status() - soup = BeautifulSoup(response.text, "html.parser") - - # page_identifier is used to check what type of response page we are working with. - page_identifier = soup.find(attrs={"name": "top"}) - - # If there is a 'table' sibling, the object was found. - if page_identifier.find_next_sibling("table") is not None: - data = page_identifier.find_next_sibling( - "table").table.find_all("td") - - complete_string = "" - for string in data[1].stripped_strings: - complete_string += string + " " - match = re.compile(r"Classification: ([\S\s]+) SPK-ID: (\d+)").match( - complete_string - ) - if match: - self.spk_id = match.group(2) - return match.group(2) - - # If there is a 'center' sibling, it is a page with a list of possible objects - elif page_identifier.find_next_sibling("center") is not None: - object_list = page_identifier.find_next_sibling("center").table.find_all( - "td" - ) - bodies = "" - obj_num = min(len(object_list), 3) - for body in object_list[:obj_num]: - bodies += body.string + "\n" - raise ValueError( - str(len(object_list)) + " different bodies found:\n" + bodies - ) + def read_headers(self): + """Read `DASTCOM5` headers and return asteroid and comet headers. + + Headers are two numpy arrays with custom dtype. + + Returns + ------- + ast_header, com_header : tuple (numpy.ndarray) + DASTCOM5 headers. + + """ + + ast_path = os.path.join(self.dbs_path, "dast5_le.dat") + ast_dtype = np.dtype( + [ + ("IBIAS1", np.int32), + ("BEGINP1", "|S8"), + ("BEGINP2", "|S8"), + ("BEGINP3", "|S8"), + ("ENDPT1", "|S8"), + ("ENDPT2", "|S8"), + ("ENDPT3", "|S8"), + ("CALDATE", "|S19"), + ("JDDATE", np.float64), + ("FTYP", "|S1"), + ("BYTE2A", np.int16), + ("IBIAS0", np.int32), + ] + ) + + with open(ast_path, "rb") as f: + ast_header = np.fromfile(f, dtype=ast_dtype, count=1) + + com_path = os.path.join(self.dbs_path, "dcom5_le.dat") + com_dtype = np.dtype( + [ + ("IBIAS2", np.int32), + ("BEGINP1", "|S8"), + ("BEGINP2", "|S8"), + ("BEGINP3", "|S8"), + ("ENDPT1", "|S8"), + ("ENDPT2", "|S8"), + ("ENDPT3", "|S8"), + ("CALDATE", "|S19"), + ("JDDATE", np.float64), + ("FTYP", "|S1"), + ("BYTE2C", np.int16), + ] + ) + + with open(com_path, "rb") as f: + com_header = np.fromfile(f, dtype=com_dtype, count=1) + + return ast_header, com_header - # If everything else failed - raise ValueError("Object could not be found. You can visit: " + - SBDB_URL + "?sstr=" + name + " for more information.") - def from_spk_id(self, spk_id, cache=True): - """Return :py:class:`~astropy.table.Table` given a SPK-ID. + def entire_db(self): + """Return complete DASTCOM5 database. - Retrieve info from NASA NeoWS API, and therefore - it only works with NEAs (Near Earth Asteroids). + Merge asteroid and comet databases, only with fields + related to orbital data, discarding the rest. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + ast_database = self.asteroid_db() + com_database = self.comet_db() + + ast_database = pd.DataFrame( + ast_database[ + list(ast_database.dtype.names[:17]) + + list(ast_database.dtype.names[-4:-3]) + + list(ast_database.dtype.names[-2:]) + ] + ) + ast_database.rename( + columns={"ASTNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True + ) + com_database = pd.DataFrame( + com_database[ + list(com_database.dtype.names[:17]) + + list(com_database.dtype.names[-4:-3]) + + list(com_database.dtype.names[-2:]) + ] + ) + com_database.rename( + columns={"COMNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True + ) + df = ast_database.append(com_database, ignore_index=True) + df[["NAME", "DESIG", "IREF"]] = df[["NAME", "DESIG", "IREF"]].apply( + lambda x: x.str.strip().str.decode("utf-8") + ) + return df + + + def read_record(self, record): + """Read `DASTCOM5` record and return body data. + + Body data consists of numpy array with custom dtype. Parameters ---------- - spk_id : str - SPK-ID number, which is given to each body by JPL. + record : int + Body record. Returns ------- - Table : ~astropy.table.Table - NEA orbit parameters. + body_data : numpy.ndarray + Body information. """ - payload = {"api_key": self.api_key or DEFAULT_API_KEY} + ast_header, com_header = self.read_headers() + ast_path = os.path.join(self.dbs_path, "dast5_le.dat") + com_path = os.path.join(self.dbs_path, "dcom5_le.dat") + # ENDPT1 indicates end of numbered asteroids records + if record <= int(ast_header["ENDPT2"][0].item()): + # ENDPT2 indicates end of unnumbered asteroids records + if record <= int(ast_header["ENDPT1"][0].item()): + # phis_rec = record_size * (record_number - IBIAS - 1 (header record)) + phis_rec = 835 * (record - ast_header["IBIAS0"][0].item() - 1) + else: + phis_rec = 835 * (record - ast_header["IBIAS1"][0].item() - 1) + + with open(ast_path, "rb") as f: + f.seek(phis_rec, os.SEEK_SET) + body_data = np.fromfile(f, dtype=self.ast_dtype, count=1) + else: + phis_rec = 976 * (record - com_header["IBIAS2"][0].item() - 1) + with open(com_path, "rb") as f: + f.seek(phis_rec, os.SEEK_SET) + body_data = np.fromfile(f, dtype=self.com_dtype, count=1) + return body_data + + def orbit_from_name(self, name): + """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a name. + + Retrieve info from JPL DASTCOM5 database. - NEOWS_URL = conf.neows_server - response = self._request( - "GET", NEOWS_URL + spk_id, params=payload, timeout=self.TIMEOUT, cache=cache - ) - response.raise_for_status() - - orbital_data = response.json()["orbital_data"] - self.name = response.json()["name"] - abs_magnitude = response.json()["absolute_magnitude_h"] - column1 = ["name", "absolute_magnitude_h"] - column2 = [self.name, abs_magnitude] - column1.extend(orbital_data.keys()) - column2.extend(orbital_data.values()) - data = Table([column1, column2], names=("Parameters", "Values")) - self.epoch = Time( - float(orbital_data["epoch_osculation"]), format="jd", scale="tdb" - ) + Parameters + ---------- + name : str + NEO name. + + Returns + ------- + Table : ~astropy.table.Table + Near Earth Asteroid/Comet orbit parameters, all stacked. + + """ + records = self.record_from_name(name) + table = self.orbit_from_record(records[0]) + for i in range(1, len(records)): + table = vstack([table, self.orbit_from_record(records[i])[1]]) + return table + + + def orbit_from_record(self, record): + """Return :py:class:`~astropy.table.Table` given a record. + + Retrieve info from JPL DASTCOM5 database. + + Parameters + ---------- + record : int + Object record. + + Returns + ------- + Table : ~astropy.table.Table + Near Earth Asteroid/Comet orbit parameters. + + """ + body_data = self.read_record(record) + a = body_data["A"].item() + ecc = body_data["EC"].item() + inc = body_data["IN"].item() + raan = body_data["OM"].item() + argp = body_data["W"].item() + m = body_data["MA"].item() + epoch = Time(body_data["EPOCH"].item(), format="jd", scale="tdb") + column2 = (record, a, ecc, inc, raan, argp, m, epoch) + column1 = ("record", "a", "ecc", "inc", "raan", "argp", "m", "EPOCH") + data = Table(rows=[column1, column2]) return data - def from_name(self, name, cache=True): - if not self.spk_id: - self._spk_id_from_name(name=name, cache=cache) - return self.from_spk_id(spk_id=self.spk_id, cache=cache) + + def record_from_name(self, name): + """Search `dastcom.idx` and return logical records that match a given string. + + Body name, SPK-ID, or alternative designations can be used. + + Parameters + ---------- + name : str + Body name. + + Returns + ------- + records : list (int) + DASTCOM5 database logical records matching str. + + """ + records = [] + lines = self.string_record_from_name(name) + for line in lines: + records.append(int(line[:6].lstrip())) + return records + + + def string_record_from_name(self, name): + """Search `dastcom.idx` and return body full record. + + Search DASTCOM5 index and return body records that match string, + containing logical record, name, alternative designations, SPK-ID, etc. + + Parameters + ---------- + name : str + Body name. + + Returns + ------- + lines: list(str) + Body records + """ + + idx_path = os.path.join(self.dbs_path, "dastcom.idx") + lines = [] + with open(idx_path, "r") as inF: + for line in inF: + if re.search(r"\b" + name.casefold() + r"\b", line.casefold()): + lines.append(line) + return lines -Neows = NeowsClass() +Dastcom5 = Dastcom5Class() From 6633d0ce6371733d14ab3a1fd25e08032d6b6ddc Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 17 Jan 2019 22:55:18 +0530 Subject: [PATCH 05/28] Complete the DASTCOM5 Module, Add tests --- astroquery/dastcom5/core.py | 18 +--- astroquery/dastcom5/tests/__init__.py | 0 astroquery/dastcom5/tests/test_dastcom5.py | 106 +++++++++++++++++++++ 3 files changed, 111 insertions(+), 13 deletions(-) create mode 100644 astroquery/dastcom5/tests/__init__.py create mode 100644 astroquery/dastcom5/tests/test_dastcom5.py diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index b6e90f3916..0b5c2f0ab8 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -12,6 +12,7 @@ from ..query import BaseQuery from ..utils import async_to_sync + @async_to_sync class Dastcom5Class(BaseQuery): """ @@ -36,11 +37,10 @@ def __init__(self, spk_id=None, name=None): self.ast_dtype = conf.AST_DTYPE self.ftp_url = conf.FTP_DB_URL - def download_dastcom5(self): """Downloads DASTCOM5 database. - Downloads and unzip DASTCOM5 file in default poliastro path (~/.poliastro). + Downloads and unzip DASTCOM5 file in default astroquery path (~/.astroquery). """ @@ -49,7 +49,8 @@ def download_dastcom5(self): if os.path.isdir(dastcom5_dir): raise FileExistsError( - "dastcom5 is already created in " + os.path.abspath(dastcom5_dir) + "dastcom5 is already created in " + \ + os.path.abspath(dastcom5_dir) ) if not zipfile.is_zipfile(dastcom5_zip_path): if not os.path.isdir(self.local_path): @@ -66,7 +67,6 @@ def _show_download_progress(self, transferred, block, totalsize): total_mb = totalsize / (1024 * 1024) print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) - def asteroid_db(self): """Return complete DASTCOM5 asteroid database. @@ -81,8 +81,6 @@ def asteroid_db(self): data = np.fromfile(f, dtype=self.ast_dtype) return data - - def comet_db(self): """Return complete DASTCOM5 comet database. @@ -97,7 +95,6 @@ def comet_db(self): data = np.fromfile(f, dtype=self.com_dtype) return data - def read_headers(self): """Read `DASTCOM5` headers and return asteroid and comet headers. @@ -153,7 +150,6 @@ def read_headers(self): return ast_header, com_header - def entire_db(self): """Return complete DASTCOM5 database. @@ -195,7 +191,6 @@ def entire_db(self): ) return df - def read_record(self, record): """Read `DASTCOM5` record and return body data. @@ -235,7 +230,7 @@ def read_record(self, record): return body_data def orbit_from_name(self, name): - """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a name. + """Return :py:class:`~astropy.table.Table` given a name. Retrieve info from JPL DASTCOM5 database. @@ -256,7 +251,6 @@ def orbit_from_name(self, name): table = vstack([table, self.orbit_from_record(records[i])[1]]) return table - def orbit_from_record(self, record): """Return :py:class:`~astropy.table.Table` given a record. @@ -287,7 +281,6 @@ def orbit_from_record(self, record): return data - def record_from_name(self, name): """Search `dastcom.idx` and return logical records that match a given string. @@ -310,7 +303,6 @@ def record_from_name(self, name): records.append(int(line[:6].lstrip())) return records - def string_record_from_name(self, name): """Search `dastcom.idx` and return body full record. diff --git a/astroquery/dastcom5/tests/__init__.py b/astroquery/dastcom5/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py new file mode 100644 index 0000000000..5151a89079 --- /dev/null +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -0,0 +1,106 @@ +import os +from unittest import mock + +import numpy as np +import pytest + +import ...dastcom5 +from ...dastcom5 import Dastcom5 + +from astropy.table import Table + + +@mock.patch("astroquery.dastcom5.np.fromfile") +@mock.patch("astroquery.dastcom5.open") +def test_asteroid_db_is_called_with_right_path(mock_open, mock_np_fromfile): + Dastcom5.asteroid_db() + mock_open.assert_called_with(Dastcom5.ast_path, "rb") + + +@mock.patch("astroquery.dastcom5.np.fromfile") +@mock.patch("astroquery.dastcom5.open") +def test_comet_db_is_called_with_right_path(mock_open, mock_np_fromfile): + Dastcom5.comet_db() + mock_open.assert_called_with(Dastcom5.com_path, "rb") + + +@mock.patch("astroquery.dastcom5.np.fromfile") +@mock.patch("astroquery.dastcom5.open") +def test_read_headers(mock_open, mock_np_fromfile): + Dastcom5.read_headers() + mock_open.assert_any_call( + os.path.join(Dastcom5.dbs_path, "dast5_le.dat"), "rb" + ) + mock_open.assert_any_call( + os.path.join(Dastcom5.dbs_path, "dcom5_le.dat"), "rb" + ) + + +@mock.patch("astroquery.dastcom5.Dastcom5.read_headers") +@mock.patch("astroquery.dastcom5.np.fromfile") +@mock.patch("astroquery.dastcom5.open") +def test_read_record(mock_open, mock_np_fromfile, mock_read_headers): + mocked_ast_headers = np.array( + [(3184, -1, b"00740473", b"00496815")], + dtype=[ + ("IBIAS1", np.int32), + ("IBIAS0", np.int32), + ("ENDPT2", "|S8"), + ("ENDPT1", "|S8"), + ], + ) + mocked_com_headers = np.array([(99999,)], dtype=[("IBIAS2", " Date: Thu, 17 Jan 2019 23:02:48 +0530 Subject: [PATCH 06/28] Fix PEP8 Errors --- astroquery/dastcom5/core.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 0b5c2f0ab8..2fa75413bb 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -49,8 +49,8 @@ def download_dastcom5(self): if os.path.isdir(dastcom5_dir): raise FileExistsError( - "dastcom5 is already created in " + \ - os.path.abspath(dastcom5_dir) + "dastcom5 is already created in " + + os.path.abspath(dastcom5_dir) ) if not zipfile.is_zipfile(dastcom5_zip_path): if not os.path.isdir(self.local_path): @@ -167,9 +167,9 @@ def entire_db(self): ast_database = pd.DataFrame( ast_database[ - list(ast_database.dtype.names[:17]) - + list(ast_database.dtype.names[-4:-3]) - + list(ast_database.dtype.names[-2:]) + list(ast_database.dtype.names[:17]) + + list(ast_database.dtype.names[-4:-3]) + + list(ast_database.dtype.names[-2:]) ] ) ast_database.rename( @@ -177,9 +177,9 @@ def entire_db(self): ) com_database = pd.DataFrame( com_database[ - list(com_database.dtype.names[:17]) - + list(com_database.dtype.names[-4:-3]) - + list(com_database.dtype.names[-2:]) + list(com_database.dtype.names[:17]) + + list(com_database.dtype.names[-4:-3]) + + list(com_database.dtype.names[-2:]) ] ) com_database.rename( From 01898ca30f9e0362937c807323bf4e114b66e28f Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Mon, 28 Jan 2019 10:16:08 +0530 Subject: [PATCH 07/28] Minor changes --- astroquery/dastcom5/core.py | 1 + astroquery/dastcom5/tests/test_dastcom5.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 2fa75413bb..fedcef7036 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -3,6 +3,7 @@ import urllib.request import zipfile import numpy as np +import pandas as pd from astropy.time import Time from astropy.table import Table, vstack diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py index 5151a89079..d695a0ab52 100644 --- a/astroquery/dastcom5/tests/test_dastcom5.py +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -4,7 +4,6 @@ import numpy as np import pytest -import ...dastcom5 from ...dastcom5 import Dastcom5 from astropy.table import Table From d3f26ffd18a19eb31efac516278dde6d7c096e42 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Mon, 28 Jan 2019 10:27:37 +0530 Subject: [PATCH 08/28] Add Python2 Support --- astroquery/dastcom5/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index fedcef7036..59cd0d0c1d 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -1,3 +1,4 @@ +from __future__ import print_function import re import os import urllib.request From 8064ab673ea3f648ec4f9e6a607b6b30d146c01b Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Wed, 30 Jan 2019 00:12:02 +0530 Subject: [PATCH 09/28] Work on suggestions --- astroquery/dastcom5/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 59cd0d0c1d..0977a448a7 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -268,7 +268,7 @@ def orbit_from_record(self, record): Table : ~astropy.table.Table Near Earth Asteroid/Comet orbit parameters. - """ + """ body_data = self.read_record(record) a = body_data["A"].item() ecc = body_data["EC"].item() @@ -320,8 +320,8 @@ def string_record_from_name(self, name): ------- lines: list(str) Body records - """ + """ idx_path = os.path.join(self.dbs_path, "dastcom.idx") lines = [] with open(idx_path, "r") as inF: From c973e3ef3373c919a3707ed93e78fb5bee41a785 Mon Sep 17 00:00:00 2001 From: Ayush Suhane Date: Tue, 5 Feb 2019 22:25:17 +0530 Subject: [PATCH 10/28] Fix download progress issue --- astroquery/dastcom5/core.py | 20 ++++--- astroquery/dastcom5/tests/test_dastcom5.py | 66 ++++++++++------------ 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 0977a448a7..d795a80eca 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -5,6 +5,7 @@ import zipfile import numpy as np import pandas as pd +from tqdm import tqdm from astropy.time import Time from astropy.table import Table, vstack @@ -38,6 +39,7 @@ def __init__(self, spk_id=None, name=None): self.com_dtype = conf.COM_DTYPE self.ast_dtype = conf.AST_DTYPE self.ftp_url = conf.FTP_DB_URL + self._show_download_progress = self._Show_Download_Progress def download_dastcom5(self): """Downloads DASTCOM5 database. @@ -58,16 +60,20 @@ def download_dastcom5(self): if not os.path.isdir(self.local_path): os.makedirs(self.local_path) - urllib.request.urlretrieve( - self.ftp_url + "dastcom5.zip", dastcom5_zip_path, self._show_download_progress - ) + print("Downloading datscom5.zip") + with self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip") as t: + urllib.request.urlretrieve(self.ftp_url + "dastcom5.zip", filename=dastcom5_zip_path, reporthook=t.update_to) + with zipfile.ZipFile(dastcom5_zip_path) as myzip: myzip.extractall(self.local_path) - def _show_download_progress(self, transferred, block, totalsize): - trans_mb = transferred * block / (1024 * 1024) - total_mb = totalsize / (1024 * 1024) - print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) + class _Show_Download_Progress(tqdm): + """Helper class for displaying download progress bar when using download_dastcom5() function + """ + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) def asteroid_db(self): """Return complete DASTCOM5 asteroid database. diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py index d695a0ab52..36e163f883 100644 --- a/astroquery/dastcom5/tests/test_dastcom5.py +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -1,31 +1,31 @@ import os -from unittest import mock import numpy as np import pytest +from pytest_mock import mocker from ...dastcom5 import Dastcom5 from astropy.table import Table -@mock.patch("astroquery.dastcom5.np.fromfile") -@mock.patch("astroquery.dastcom5.open") -def test_asteroid_db_is_called_with_right_path(mock_open, mock_np_fromfile): +def test_asteroid_db_is_called_with_right_path(mocker): + mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") + mock_open = mocker.patch("poliastro.neos.dastcom5.open") Dastcom5.asteroid_db() mock_open.assert_called_with(Dastcom5.ast_path, "rb") -@mock.patch("astroquery.dastcom5.np.fromfile") -@mock.patch("astroquery.dastcom5.open") -def test_comet_db_is_called_with_right_path(mock_open, mock_np_fromfile): +def test_comet_db_is_called_with_right_path(mocker): + mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") + mock_open = mocker.patch("poliastro.neos.dastcom5.open") Dastcom5.comet_db() mock_open.assert_called_with(Dastcom5.com_path, "rb") -@mock.patch("astroquery.dastcom5.np.fromfile") -@mock.patch("astroquery.dastcom5.open") -def test_read_headers(mock_open, mock_np_fromfile): +def test_read_headers(mocker): + mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") + mock_open = mocker.patch("poliastro.neos.dastcom5.open") Dastcom5.read_headers() mock_open.assert_any_call( os.path.join(Dastcom5.dbs_path, "dast5_le.dat"), "rb" @@ -35,10 +35,10 @@ def test_read_headers(mock_open, mock_np_fromfile): ) -@mock.patch("astroquery.dastcom5.Dastcom5.read_headers") -@mock.patch("astroquery.dastcom5.np.fromfile") -@mock.patch("astroquery.dastcom5.open") -def test_read_record(mock_open, mock_np_fromfile, mock_read_headers): +def test_read_record(mocker): + mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") + mock_open = mocker.patch("poliastro.neos.dastcom5.open") + mock_read_headers = mocker.patch("astroquery.dastcom5.Dastcom5.read_headers") mocked_ast_headers = np.array( [(3184, -1, b"00740473", b"00496815")], dtype=[ @@ -61,13 +61,11 @@ def test_read_record(mock_open, mock_np_fromfile, mock_read_headers): ) -@mock.patch("astroquery.dastcom5.os.makedirs") -@mock.patch("astroquery.dastcom5.zipfile") -@mock.patch("astroquery.dastcom5.os.path.isdir") -@mock.patch("astroquery.dastcom5.urllib.request") -def test_download_dastcom5_raises_error_when_folder_exists( - mock_request, mock_isdir, mock_zipfile, mock_makedirs -): +def test_download_dastcom5_raises_error_when_folder_exists(mocker): + mock_request = mocker.patch("astroquery.dastcom5.urllib.request") + mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") + mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") + mock_makedirs = mocker.patch("astroquery.dastcom5.os.makedirs") mock_isdir.side_effect = lambda x: x == os.path.join( Dastcom5.local_path, "dastcom5" ) @@ -78,28 +76,26 @@ def test_download_dastcom5_raises_error_when_folder_exists( ) -@mock.patch("astroquery.dastcom5.urllib.request") -@mock.patch("astroquery.dastcom5.os.makedirs") -@mock.patch("astroquery.dastcom5.zipfile") -@mock.patch("astroquery.dastcom5.os.path.isdir") -def test_download_dastcom5_creates_folder( - mock_isdir, mock_zipfile, mock_makedirs, mock_request -): +def test_download_dastcom5_creates_folder(mocker): + mock_request = mocker.patch("astroquery.dastcom5.urllib.request") + mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") + mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") + mock_makedirs = mocker.patch("astroquery.dastcom5.os.makedirs") mock_isdir.return_value = False mock_zipfile.is_zipfile.return_value = False Dastcom5.download_dastcom5() mock_makedirs.assert_called_once_with(Dastcom5.local_path) -@mock.patch("astroquery.dastcom5.zipfile") -@mock.patch("astroquery.dastcom5.os.path.isdir") -@mock.patch("astroquery.dastcom5.urllib.request.urlretrieve") -def test_download_dastcom5_downloads_file(mock_request, mock_isdir, mock_zipfile): +def test_download_dastcom5_downloads_file(mocker): + mock_request = mocker.patch("astroquery.dastcom5.urllib.request.urlretrieve") + mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") + mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") mock_isdir.side_effect = lambda x: x == Dastcom5.local_path mock_zipfile.is_zipfile.return_value = False Dastcom5.download_dastcom5() mock_request.assert_called_once_with( - Dastcom5.FTP_DB_URL + "dastcom5.zip", - os.path.join(Dastcom5.local_path, "dastcom5.zip"), - Dastcom5._show_download_progress, + Dastcom5.ftp_url + "dastcom5.zip", + filename=os.path.join(Dastcom5.local_path, "dastcom5.zip"), + reporthook=Dastcom5.self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip").update_to, ) From 8d853fdc806b81f0d9fe32bccb36b23fe92d206f Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Tue, 5 Feb 2019 22:27:11 +0530 Subject: [PATCH 11/28] Remove vstack --- astroquery/dastcom5/core.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index d795a80eca..c9e886c170 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -8,7 +8,7 @@ from tqdm import tqdm from astropy.time import Time -from astropy.table import Table, vstack +from astropy.table import Table import astropy.units as u from . import conf @@ -254,10 +254,9 @@ def orbit_from_name(self, name): """ records = self.record_from_name(name) - table = self.orbit_from_record(records[0]) - for i in range(1, len(records)): - table = vstack([table, self.orbit_from_record(records[i])[1]]) - return table + orbits = [self.orbit_from_record(rec) for rec in records] + tbl = Table(orbits) + return tbl def orbit_from_record(self, record): """Return :py:class:`~astropy.table.Table` given a record. From d34b2fc8e0bb31e4570328d44b0cf00b0aba06c1 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Tue, 5 Feb 2019 22:29:32 +0530 Subject: [PATCH 12/28] Fix PEP8 issues! --- astroquery/dastcom5/core.py | 3 ++- astroquery/dastcom5/tests/test_dastcom5.py | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index c9e886c170..5109eb576b 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -62,7 +62,8 @@ def download_dastcom5(self): print("Downloading datscom5.zip") with self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip") as t: - urllib.request.urlretrieve(self.ftp_url + "dastcom5.zip", filename=dastcom5_zip_path, reporthook=t.update_to) + urllib.request.urlretrieve( + self.ftp_url + "dastcom5.zip", filename=dastcom5_zip_path, reporthook=t.update_to) with zipfile.ZipFile(dastcom5_zip_path) as myzip: myzip.extractall(self.local_path) diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py index 36e163f883..b8045c5046 100644 --- a/astroquery/dastcom5/tests/test_dastcom5.py +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -38,7 +38,8 @@ def test_read_headers(mocker): def test_read_record(mocker): mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") mock_open = mocker.patch("poliastro.neos.dastcom5.open") - mock_read_headers = mocker.patch("astroquery.dastcom5.Dastcom5.read_headers") + mock_read_headers = mocker.patch( + "astroquery.dastcom5.Dastcom5.read_headers") mocked_ast_headers = np.array( [(3184, -1, b"00740473", b"00496815")], dtype=[ @@ -88,7 +89,8 @@ def test_download_dastcom5_creates_folder(mocker): def test_download_dastcom5_downloads_file(mocker): - mock_request = mocker.patch("astroquery.dastcom5.urllib.request.urlretrieve") + mock_request = mocker.patch( + "astroquery.dastcom5.urllib.request.urlretrieve") mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") mock_isdir.side_effect = lambda x: x == Dastcom5.local_path @@ -97,5 +99,6 @@ def test_download_dastcom5_downloads_file(mocker): mock_request.assert_called_once_with( Dastcom5.ftp_url + "dastcom5.zip", filename=os.path.join(Dastcom5.local_path, "dastcom5.zip"), - reporthook=Dastcom5.self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip").update_to, + reporthook=Dastcom5.self._show_download_progress( + unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip").update_to, ) From b0f3f07a850a0f5a0f1c37b45082efb368b285bf Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Wed, 6 Feb 2019 22:05:06 +0530 Subject: [PATCH 13/28] Remove urllib, remove class, add _download_file --- astroquery/dastcom5/core.py | 16 ++-------------- astroquery/dastcom5/tests/test_dastcom5.py | 12 +++++------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 5109eb576b..21a5231081 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -1,11 +1,9 @@ from __future__ import print_function import re import os -import urllib.request import zipfile import numpy as np import pandas as pd -from tqdm import tqdm from astropy.time import Time from astropy.table import Table @@ -39,7 +37,6 @@ def __init__(self, spk_id=None, name=None): self.com_dtype = conf.COM_DTYPE self.ast_dtype = conf.AST_DTYPE self.ftp_url = conf.FTP_DB_URL - self._show_download_progress = self._Show_Download_Progress def download_dastcom5(self): """Downloads DASTCOM5 database. @@ -50,6 +47,7 @@ def download_dastcom5(self): dastcom5_dir = os.path.join(self.local_path, "dastcom5") dastcom5_zip_path = os.path.join(self.local_path, "dastcom5.zip") + ftp_path = self.ftp_url + "dastcom5.zip" if os.path.isdir(dastcom5_dir): raise FileExistsError( @@ -61,21 +59,11 @@ def download_dastcom5(self): os.makedirs(self.local_path) print("Downloading datscom5.zip") - with self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip") as t: - urllib.request.urlretrieve( - self.ftp_url + "dastcom5.zip", filename=dastcom5_zip_path, reporthook=t.update_to) + self._download_file(url=ftp_path, local_filepath=dastcom5_zip_path) with zipfile.ZipFile(dastcom5_zip_path) as myzip: myzip.extractall(self.local_path) - class _Show_Download_Progress(tqdm): - """Helper class for displaying download progress bar when using download_dastcom5() function - """ - def update_to(self, b=1, bsize=1, tsize=None): - if tsize is not None: - self.total = tsize - self.update(b * bsize - self.n) - def asteroid_db(self): """Return complete DASTCOM5 asteroid database. diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py index b8045c5046..37b67d8dc8 100644 --- a/astroquery/dastcom5/tests/test_dastcom5.py +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -89,16 +89,14 @@ def test_download_dastcom5_creates_folder(mocker): def test_download_dastcom5_downloads_file(mocker): - mock_request = mocker.patch( - "astroquery.dastcom5.urllib.request.urlretrieve") + mock_download = mocker.patch( + "astroquery.query.BaseQuery._download_file") mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") mock_isdir.side_effect = lambda x: x == Dastcom5.local_path mock_zipfile.is_zipfile.return_value = False Dastcom5.download_dastcom5() - mock_request.assert_called_once_with( - Dastcom5.ftp_url + "dastcom5.zip", - filename=os.path.join(Dastcom5.local_path, "dastcom5.zip"), - reporthook=Dastcom5.self._show_download_progress( - unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip").update_to, + mock_download.assert_called_once_with( + url=Dastcom5.ftp_url + "dastcom5.zip", + local_filepath=os.path.join(Dastcom5.local_path, "dastcom5.zip"), ) From 9d479d6b0b1e283ac6197721ba196925f77cd6b8 Mon Sep 17 00:00:00 2001 From: Antonio Hidalgo Date: Thu, 17 Jan 2019 20:25:45 +0530 Subject: [PATCH 14/28] Add the dastcom5 module from poliastro --- astroquery/dastcom5/__init__.py | 13 + astroquery/dastcom5/dastcom5.py | 591 ++++++++++++++++++++++++++++++++ 2 files changed, 604 insertions(+) create mode 100644 astroquery/dastcom5/__init__.py create mode 100644 astroquery/dastcom5/dastcom5.py diff --git a/astroquery/dastcom5/__init__.py b/astroquery/dastcom5/__init__.py new file mode 100644 index 0000000000..59c3d87ddc --- /dev/null +++ b/astroquery/dastcom5/__init__.py @@ -0,0 +1,13 @@ +"""Code related to NEOs. + +Functions related to NEOs and different NASA APIs. +All of them are coded as part of SOCIS 2017 proposal. + +Notes +----- + +The orbits returned by the functions in this package are in the +:py:class:`~poliastro.frames.HeliocentricEclipticJ2000` frame. + +""" +from poliastro.neos import neows, dastcom5 # flake8: noqa diff --git a/astroquery/dastcom5/dastcom5.py b/astroquery/dastcom5/dastcom5.py new file mode 100644 index 0000000000..6baa7ee394 --- /dev/null +++ b/astroquery/dastcom5/dastcom5.py @@ -0,0 +1,591 @@ +"""NEOs orbit from DASTCOM5 database. + +""" +import os +import re +import urllib.request +import zipfile + +import astropy.units as u +import numpy as np +import pandas as pd +from astropy.time import Time + +from poliastro.bodies import Sun +from poliastro.frames import HeliocentricEclipticJ2000 +from poliastro.twobody.angles import M_to_nu +from poliastro.twobody.orbit import Orbit + +AST_DTYPE = np.dtype( + [ + ("NO", np.int32), + ("NOBS", np.int32), + ("OBSFRST", np.int32), + ("OBSLAST", np.int32), + ("EPOCH", np.float64), + ("CALEPO", np.float64), + ("MA", np.float64), + ("W", np.float64), + ("OM", np.float64), + ("IN", np.float64), + ("EC", np.float64), + ("A", np.float64), + ("QR", np.float64), + ("TP", np.float64), + ("TPCAL", np.float64), + ("TPFRAC", np.float64), + ("SOLDAT", np.float64), + ("SRC1", np.float64), + ("SRC2", np.float64), + ("SRC3", np.float64), + ("SRC4", np.float64), + ("SRC5", np.float64), + ("SRC6", np.float64), + ("SRC7", np.float64), + ("SRC8", np.float64), + ("SRC9", np.float64), + ("SRC10", np.float64), + ("SRC11", np.float64), + ("SRC12", np.float64), + ("SRC13", np.float64), + ("SRC14", np.float64), + ("SRC15", np.float64), + ("SRC16", np.float64), + ("SRC17", np.float64), + ("SRC18", np.float64), + ("SRC19", np.float64), + ("SRC20", np.float64), + ("SRC21", np.float64), + ("SRC22", np.float64), + ("SRC23", np.float64), + ("SRC24", np.float64), + ("SRC25", np.float64), + ("SRC26", np.float64), + ("SRC27", np.float64), + ("SRC28", np.float64), + ("SRC29", np.float64), + ("SRC30", np.float64), + ("SRC31", np.float64), + ("SRC32", np.float64), + ("SRC33", np.float64), + ("SRC34", np.float64), + ("SRC35", np.float64), + ("SRC36", np.float64), + ("SRC37", np.float64), + ("SRC38", np.float64), + ("SRC39", np.float64), + ("SRC40", np.float64), + ("SRC41", np.float64), + ("SRC42", np.float64), + ("SRC43", np.float64), + ("SRC44", np.float64), + ("SRC45", np.float64), + ("PRELTV", np.int8), + ("SPHMX3", np.int8), + ("SPHMX5", np.int8), + ("JGSEP", np.int8), + ("TWOBOD", np.int8), + ("NSATS", np.int8), + ("UPARM", np.int8), + ("LSRC", np.int8), + ("NDEL", np.int16), + ("NDOP", np.int16), + ("H", np.float32), + ("G", np.float32), + ("A1", np.float32), + ("A2", np.float32), + ("A3", np.float32), + ("R0", np.float32), + ("ALN", np.float32), + ("NM", np.float32), + ("NN", np.float32), + ("NK", np.float32), + ("LGK", np.float32), + ("RHO", np.float32), + ("AMRAT", np.float32), + ("ALF", np.float32), + ("DEL", np.float32), + ("SPHLM3", np.float32), + ("SPHLM5", np.float32), + ("RP", np.float32), + ("GM", np.float32), + ("RAD", np.float32), + ("EXTNT1", np.float32), + ("EXTNT2", np.float32), + ("EXTNT3", np.float32), + ("MOID", np.float32), + ("ALBEDO", np.float32), + ("BVCI", np.float32), + ("UBCI", np.float32), + ("IRCI", np.float32), + ("RMSW", np.float32), + ("RMSU", np.float32), + ("RMSN", np.float32), + ("RMSNT", np.float32), + ("RMSH", np.float32), + ("EQUNOX", "|S4"), + ("PENAM", "|S6"), + ("SBNAM", "|S12"), + ("SPTYPT", "|S5"), + ("SPTYPS", "|S5"), + ("DARC", "|S9"), + ("COMNT1", "|S41"), + ("COMNT2", "|S80"), + ("DESIG", "|S13"), + ("ASTEST", "|S8"), + ("IREF", "|S10"), + ("ASTNAM", "|S18"), + ] +) + +COM_DTYPE = np.dtype( + [ + ("NO", np.int32), + ("NOBS", np.int32), + ("OBSFRST", np.int32), + ("OBSLAST", np.int32), + ("EPOCH", np.float64), + ("CALEPO", np.float64), + ("MA", np.float64), + ("W", np.float64), + ("OM", np.float64), + ("IN", np.float64), + ("EC", np.float64), + ("A", np.float64), + ("QR", np.float64), + ("TP", np.float64), + ("TPCAL", np.float64), + ("TPFRAC", np.float64), + ("SOLDAT", np.float64), + ("SRC1", np.float64), + ("SRC2", np.float64), + ("SRC3", np.float64), + ("SRC4", np.float64), + ("SRC5", np.float64), + ("SRC6", np.float64), + ("SRC7", np.float64), + ("SRC8", np.float64), + ("SRC9", np.float64), + ("SRC10", np.float64), + ("SRC11", np.float64), + ("SRC12", np.float64), + ("SRC13", np.float64), + ("SRC14", np.float64), + ("SRC15", np.float64), + ("SRC16", np.float64), + ("SRC17", np.float64), + ("SRC18", np.float64), + ("SRC19", np.float64), + ("SRC20", np.float64), + ("SRC21", np.float64), + ("SRC22", np.float64), + ("SRC23", np.float64), + ("SRC24", np.float64), + ("SRC25", np.float64), + ("SRC26", np.float64), + ("SRC27", np.float64), + ("SRC28", np.float64), + ("SRC29", np.float64), + ("SRC30", np.float64), + ("SRC31", np.float64), + ("SRC32", np.float64), + ("SRC33", np.float64), + ("SRC34", np.float64), + ("SRC35", np.float64), + ("SRC36", np.float64), + ("SRC37", np.float64), + ("SRC38", np.float64), + ("SRC39", np.float64), + ("SRC40", np.float64), + ("SRC41", np.float64), + ("SRC42", np.float64), + ("SRC43", np.float64), + ("SRC44", np.float64), + ("SRC45", np.float64), + ("SRC46", np.float64), + ("SRC47", np.float64), + ("SRC48", np.float64), + ("SRC49", np.float64), + ("SRC50", np.float64), + ("SRC51", np.float64), + ("SRC52", np.float64), + ("SRC53", np.float64), + ("SRC54", np.float64), + ("SRC55", np.float64), + ("PRELTV", np.int8), + ("SPHMX3", np.int8), + ("SPHMX5", np.int8), + ("JGSEP", np.int8), + ("TWOBOD", np.int8), + ("NSATS", np.int8), + ("UPARM", np.int8), + ("LSRC", np.int8), + ("IPYR", np.int16), + ("NDEL", np.int16), + ("NDOP", np.int16), + ("NOBSMT", np.int16), + ("NOBSMN", np.int16), + ("H", np.float32), + ("G", np.float32), + ("M1 (MT)", np.float32), + ("M2 (MN)", np.float32), + ("K1 (MTSMT)", np.float32), + ("K2 (MNSMT)", np.float32), + ("PHCOF (MNP)", np.float32), + ("A1", np.float32), + ("A2", np.float32), + ("A3", np.float32), + ("DT", np.float32), + ("R0", np.float32), + ("ALN", np.float32), + ("NM", np.float32), + ("NN", np.float32), + ("NK", np.float32), + ("S0", np.float32), + ("TCL", np.float32), + ("RHO", np.float32), + ("AMRAT", np.float32), + ("AJ1", np.float32), + ("AJ2", np.float32), + ("ET1", np.float32), + ("ET2", np.float32), + ("DTH", np.float32), + ("ALF", np.float32), + ("DEL", np.float32), + ("SPHLM3", np.float32), + ("SPHLM5", np.float32), + ("RP", np.float32), + ("GM", np.float32), + ("RAD", np.float32), + ("EXTNT1", np.float32), + ("EXTNT2", np.float32), + ("EXTNT3", np.float32), + ("MOID", np.float32), + ("ALBEDO", np.float32), + ("RMSW", np.float32), + ("RMSU", np.float32), + ("RMSN", np.float32), + ("RMSNT", np.float32), + ("RMSMT", np.float32), + ("RMSMN", np.float32), + ("EQUNOX", "|S4"), + ("PENAM", "|S6"), + ("SBNAM", "|S12"), + ("DARC", "|S9"), + ("COMNT3", "|S49"), + ("COMNT2", "|S80"), + ("DESIG", "|S13"), + ("COMEST", "|S14"), + ("IREF", "|S10"), + ("COMNAM", "|S29"), + ] +) + +POLIASTRO_LOCAL_PATH = os.path.join(os.path.expanduser("~"), ".poliastro") +DBS_LOCAL_PATH = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5", "dat") +AST_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") +COM_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") + +FTP_DB_URL = "ftp://ssd.jpl.nasa.gov/pub/ssd/" + + +def asteroid_db(): + """Return complete DASTCOM5 asteroid database. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + with open(AST_DB_PATH, "rb") as f: + f.seek(835, os.SEEK_SET) + data = np.fromfile(f, dtype=AST_DTYPE) + return data + + + +def comet_db(): + """Return complete DASTCOM5 comet database. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + with open(COM_DB_PATH, "rb") as f: + f.seek(976, os.SEEK_SET) + data = np.fromfile(f, dtype=COM_DTYPE) + return data + + +def orbit_from_name(name): + """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a name. + + Retrieve info from JPL DASTCOM5 database. + + Parameters + ---------- + name : str + NEO name. + + Returns + ------- + orbit : list (~poliastro.twobody.orbit.Orbit) + NEO orbits. + + """ + records = record_from_name(name) + orbits = [] + for record in records: + orbits.append(orbit_from_record(record)) + return orbits + + +def orbit_from_record(record): + """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a record. + + Retrieve info from JPL DASTCOM5 database. + + Parameters + ---------- + record : int + Object record. + + Returns + ------- + orbit : ~poliastro.twobody.orbit.Orbit + NEO orbit. + + """ + body_data = read_record(record) + a = body_data["A"].item() * u.au + ecc = body_data["EC"].item() * u.one + inc = body_data["IN"].item() * u.deg + raan = body_data["OM"].item() * u.deg + argp = body_data["W"].item() * u.deg + m = body_data["MA"].item() * u.deg + nu = M_to_nu(m, ecc) + epoch = Time(body_data["EPOCH"].item(), format="jd", scale="tdb") + + orbit = Orbit.from_classical(Sun, a, ecc, inc, raan, argp, nu, epoch) + orbit._frame = HeliocentricEclipticJ2000(obstime=epoch) + return orbit + + +def record_from_name(name): + """Search `dastcom.idx` and return logical records that match a given string. + + Body name, SPK-ID, or alternative designations can be used. + + Parameters + ---------- + name : str + Body name. + + Returns + ------- + records : list (int) + DASTCOM5 database logical records matching str. + + """ + records = [] + lines = string_record_from_name(name) + for line in lines: + records.append(int(line[:6].lstrip())) + return records + + +def string_record_from_name(name): + """Search `dastcom.idx` and return body full record. + + Search DASTCOM5 index and return body records that match string, + containing logical record, name, alternative designations, SPK-ID, etc. + + Parameters + ---------- + name : str + Body name. + + Returns + ------- + lines: list(str) + Body records + """ + + idx_path = os.path.join(DBS_LOCAL_PATH, "dastcom.idx") + lines = [] + with open(idx_path, "r") as inF: + for line in inF: + if re.search(r"\b" + name.casefold() + r"\b", line.casefold()): + lines.append(line) + return lines + + +def read_headers(): + """Read `DASTCOM5` headers and return asteroid and comet headers. + + Headers are two numpy arrays with custom dtype. + + Returns + ------- + ast_header, com_header : tuple (numpy.ndarray) + DASTCOM5 headers. + + """ + + ast_path = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") + ast_dtype = np.dtype( + [ + ("IBIAS1", np.int32), + ("BEGINP1", "|S8"), + ("BEGINP2", "|S8"), + ("BEGINP3", "|S8"), + ("ENDPT1", "|S8"), + ("ENDPT2", "|S8"), + ("ENDPT3", "|S8"), + ("CALDATE", "|S19"), + ("JDDATE", np.float64), + ("FTYP", "|S1"), + ("BYTE2A", np.int16), + ("IBIAS0", np.int32), + ] + ) + + with open(ast_path, "rb") as f: + ast_header = np.fromfile(f, dtype=ast_dtype, count=1) + + com_path = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") + com_dtype = np.dtype( + [ + ("IBIAS2", np.int32), + ("BEGINP1", "|S8"), + ("BEGINP2", "|S8"), + ("BEGINP3", "|S8"), + ("ENDPT1", "|S8"), + ("ENDPT2", "|S8"), + ("ENDPT3", "|S8"), + ("CALDATE", "|S19"), + ("JDDATE", np.float64), + ("FTYP", "|S1"), + ("BYTE2C", np.int16), + ] + ) + + with open(com_path, "rb") as f: + com_header = np.fromfile(f, dtype=com_dtype, count=1) + + return ast_header, com_header + + +def read_record(record): + """Read `DASTCOM5` record and return body data. + + Body data consists of numpy array with custom dtype. + + Parameters + ---------- + record : int + Body record. + + Returns + ------- + body_data : numpy.ndarray + Body information. + + """ + ast_header, com_header = read_headers() + ast_path = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") + com_path = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") + # ENDPT1 indicates end of numbered asteroids records + if record <= int(ast_header["ENDPT2"][0].item()): + # ENDPT2 indicates end of unnumbered asteroids records + if record <= int(ast_header["ENDPT1"][0].item()): + # phis_rec = record_size * (record_number - IBIAS - 1 (header record)) + phis_rec = 835 * (record - ast_header["IBIAS0"][0].item() - 1) + else: + phis_rec = 835 * (record - ast_header["IBIAS1"][0].item() - 1) + + with open(ast_path, "rb") as f: + f.seek(phis_rec, os.SEEK_SET) + body_data = np.fromfile(f, dtype=AST_DTYPE, count=1) + else: + phis_rec = 976 * (record - com_header["IBIAS2"][0].item() - 1) + with open(com_path, "rb") as f: + f.seek(phis_rec, os.SEEK_SET) + body_data = np.fromfile(f, dtype=COM_DTYPE, count=1) + return body_data + + +def download_dastcom5(): + """Downloads DASTCOM5 database. + + Downloads and unzip DASTCOM5 file in default poliastro path (~/.poliastro). + + """ + + dastcom5_dir = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5") + dastcom5_zip_path = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5.zip") + + if os.path.isdir(dastcom5_dir): + raise FileExistsError( + "dastcom5 is already created in " + os.path.abspath(dastcom5_dir) + ) + if not zipfile.is_zipfile(dastcom5_zip_path): + if not os.path.isdir(POLIASTRO_LOCAL_PATH): + os.makedirs(POLIASTRO_LOCAL_PATH) + + urllib.request.urlretrieve( + FTP_DB_URL + "dastcom5.zip", dastcom5_zip_path, _show_download_progress + ) + with zipfile.ZipFile(dastcom5_zip_path) as myzip: + myzip.extractall(POLIASTRO_LOCAL_PATH) + + +def _show_download_progress(transferred, block, totalsize): + trans_mb = transferred * block / (1024 * 1024) + total_mb = totalsize / (1024 * 1024) + print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) + + +def entire_db(): + """Return complete DASTCOM5 database. + + Merge asteroid and comet databases, only with fields + related to orbital data, discarding the rest. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + ast_database = asteroid_db() + com_database = comet_db() + + ast_database = pd.DataFrame( + ast_database[ + list(ast_database.dtype.names[:17]) + + list(ast_database.dtype.names[-4:-3]) + + list(ast_database.dtype.names[-2:]) + ] + ) + ast_database.rename( + columns={"ASTNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True + ) + com_database = pd.DataFrame( + com_database[ + list(com_database.dtype.names[:17]) + + list(com_database.dtype.names[-4:-3]) + + list(com_database.dtype.names[-2:]) + ] + ) + com_database.rename( + columns={"COMNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True + ) + df = ast_database.append(com_database, ignore_index=True) + df[["NAME", "DESIG", "IREF"]] = df[["NAME", "DESIG", "IREF"]].apply( + lambda x: x.str.strip().str.decode("utf-8") + ) + return df From 0388788603eb997434bd13c5ddcb2bb1055983e0 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 17 Jan 2019 20:27:00 +0530 Subject: [PATCH 15/28] Add the init file --- astroquery/dastcom5/__init__.py | 298 +++++++++++++++++++++++++++++++- 1 file changed, 289 insertions(+), 9 deletions(-) mode change 100644 => 100755 astroquery/dastcom5/__init__.py diff --git a/astroquery/dastcom5/__init__.py b/astroquery/dastcom5/__init__.py old mode 100644 new mode 100755 index 59c3d87ddc..105b4e52fe --- a/astroquery/dastcom5/__init__.py +++ b/astroquery/dastcom5/__init__.py @@ -1,13 +1,293 @@ -"""Code related to NEOs. - -Functions related to NEOs and different NASA APIs. -All of them are coded as part of SOCIS 2017 proposal. +""" +DASTCOM5 Query Tool +=================== -Notes ------ +:Authors: +1) Antonio Hidalgo (antoniohidalgo.inves@gmail.com) +2) Juan Luis Cano Rodríguez (juanlu001@gmail.com) -The orbits returned by the functions in this package are in the -:py:class:`~poliastro.frames.HeliocentricEclipticJ2000` frame. +This module contains various methods for querying the +DASTCOM5. +All of the methods are coded as part of SOCIS 2017 for poliastro by Antonio Hidalgo[1]. """ -from poliastro.neos import neows, dastcom5 # flake8: noqa +from astropy import config as _config + + +class Conf(_config.ConfigNamespace): + AST_DTYPE = np.dtype( + [ + ("NO", np.int32), + ("NOBS", np.int32), + ("OBSFRST", np.int32), + ("OBSLAST", np.int32), + ("EPOCH", np.float64), + ("CALEPO", np.float64), + ("MA", np.float64), + ("W", np.float64), + ("OM", np.float64), + ("IN", np.float64), + ("EC", np.float64), + ("A", np.float64), + ("QR", np.float64), + ("TP", np.float64), + ("TPCAL", np.float64), + ("TPFRAC", np.float64), + ("SOLDAT", np.float64), + ("SRC1", np.float64), + ("SRC2", np.float64), + ("SRC3", np.float64), + ("SRC4", np.float64), + ("SRC5", np.float64), + ("SRC6", np.float64), + ("SRC7", np.float64), + ("SRC8", np.float64), + ("SRC9", np.float64), + ("SRC10", np.float64), + ("SRC11", np.float64), + ("SRC12", np.float64), + ("SRC13", np.float64), + ("SRC14", np.float64), + ("SRC15", np.float64), + ("SRC16", np.float64), + ("SRC17", np.float64), + ("SRC18", np.float64), + ("SRC19", np.float64), + ("SRC20", np.float64), + ("SRC21", np.float64), + ("SRC22", np.float64), + ("SRC23", np.float64), + ("SRC24", np.float64), + ("SRC25", np.float64), + ("SRC26", np.float64), + ("SRC27", np.float64), + ("SRC28", np.float64), + ("SRC29", np.float64), + ("SRC30", np.float64), + ("SRC31", np.float64), + ("SRC32", np.float64), + ("SRC33", np.float64), + ("SRC34", np.float64), + ("SRC35", np.float64), + ("SRC36", np.float64), + ("SRC37", np.float64), + ("SRC38", np.float64), + ("SRC39", np.float64), + ("SRC40", np.float64), + ("SRC41", np.float64), + ("SRC42", np.float64), + ("SRC43", np.float64), + ("SRC44", np.float64), + ("SRC45", np.float64), + ("PRELTV", np.int8), + ("SPHMX3", np.int8), + ("SPHMX5", np.int8), + ("JGSEP", np.int8), + ("TWOBOD", np.int8), + ("NSATS", np.int8), + ("UPARM", np.int8), + ("LSRC", np.int8), + ("NDEL", np.int16), + ("NDOP", np.int16), + ("H", np.float32), + ("G", np.float32), + ("A1", np.float32), + ("A2", np.float32), + ("A3", np.float32), + ("R0", np.float32), + ("ALN", np.float32), + ("NM", np.float32), + ("NN", np.float32), + ("NK", np.float32), + ("LGK", np.float32), + ("RHO", np.float32), + ("AMRAT", np.float32), + ("ALF", np.float32), + ("DEL", np.float32), + ("SPHLM3", np.float32), + ("SPHLM5", np.float32), + ("RP", np.float32), + ("GM", np.float32), + ("RAD", np.float32), + ("EXTNT1", np.float32), + ("EXTNT2", np.float32), + ("EXTNT3", np.float32), + ("MOID", np.float32), + ("ALBEDO", np.float32), + ("BVCI", np.float32), + ("UBCI", np.float32), + ("IRCI", np.float32), + ("RMSW", np.float32), + ("RMSU", np.float32), + ("RMSN", np.float32), + ("RMSNT", np.float32), + ("RMSH", np.float32), + ("EQUNOX", "|S4"), + ("PENAM", "|S6"), + ("SBNAM", "|S12"), + ("SPTYPT", "|S5"), + ("SPTYPS", "|S5"), + ("DARC", "|S9"), + ("COMNT1", "|S41"), + ("COMNT2", "|S80"), + ("DESIG", "|S13"), + ("ASTEST", "|S8"), + ("IREF", "|S10"), + ("ASTNAM", "|S18"), + ] + ) + + COM_DTYPE = np.dtype( + [ + ("NO", np.int32), + ("NOBS", np.int32), + ("OBSFRST", np.int32), + ("OBSLAST", np.int32), + ("EPOCH", np.float64), + ("CALEPO", np.float64), + ("MA", np.float64), + ("W", np.float64), + ("OM", np.float64), + ("IN", np.float64), + ("EC", np.float64), + ("A", np.float64), + ("QR", np.float64), + ("TP", np.float64), + ("TPCAL", np.float64), + ("TPFRAC", np.float64), + ("SOLDAT", np.float64), + ("SRC1", np.float64), + ("SRC2", np.float64), + ("SRC3", np.float64), + ("SRC4", np.float64), + ("SRC5", np.float64), + ("SRC6", np.float64), + ("SRC7", np.float64), + ("SRC8", np.float64), + ("SRC9", np.float64), + ("SRC10", np.float64), + ("SRC11", np.float64), + ("SRC12", np.float64), + ("SRC13", np.float64), + ("SRC14", np.float64), + ("SRC15", np.float64), + ("SRC16", np.float64), + ("SRC17", np.float64), + ("SRC18", np.float64), + ("SRC19", np.float64), + ("SRC20", np.float64), + ("SRC21", np.float64), + ("SRC22", np.float64), + ("SRC23", np.float64), + ("SRC24", np.float64), + ("SRC25", np.float64), + ("SRC26", np.float64), + ("SRC27", np.float64), + ("SRC28", np.float64), + ("SRC29", np.float64), + ("SRC30", np.float64), + ("SRC31", np.float64), + ("SRC32", np.float64), + ("SRC33", np.float64), + ("SRC34", np.float64), + ("SRC35", np.float64), + ("SRC36", np.float64), + ("SRC37", np.float64), + ("SRC38", np.float64), + ("SRC39", np.float64), + ("SRC40", np.float64), + ("SRC41", np.float64), + ("SRC42", np.float64), + ("SRC43", np.float64), + ("SRC44", np.float64), + ("SRC45", np.float64), + ("SRC46", np.float64), + ("SRC47", np.float64), + ("SRC48", np.float64), + ("SRC49", np.float64), + ("SRC50", np.float64), + ("SRC51", np.float64), + ("SRC52", np.float64), + ("SRC53", np.float64), + ("SRC54", np.float64), + ("SRC55", np.float64), + ("PRELTV", np.int8), + ("SPHMX3", np.int8), + ("SPHMX5", np.int8), + ("JGSEP", np.int8), + ("TWOBOD", np.int8), + ("NSATS", np.int8), + ("UPARM", np.int8), + ("LSRC", np.int8), + ("IPYR", np.int16), + ("NDEL", np.int16), + ("NDOP", np.int16), + ("NOBSMT", np.int16), + ("NOBSMN", np.int16), + ("H", np.float32), + ("G", np.float32), + ("M1 (MT)", np.float32), + ("M2 (MN)", np.float32), + ("K1 (MTSMT)", np.float32), + ("K2 (MNSMT)", np.float32), + ("PHCOF (MNP)", np.float32), + ("A1", np.float32), + ("A2", np.float32), + ("A3", np.float32), + ("DT", np.float32), + ("R0", np.float32), + ("ALN", np.float32), + ("NM", np.float32), + ("NN", np.float32), + ("NK", np.float32), + ("S0", np.float32), + ("TCL", np.float32), + ("RHO", np.float32), + ("AMRAT", np.float32), + ("AJ1", np.float32), + ("AJ2", np.float32), + ("ET1", np.float32), + ("ET2", np.float32), + ("DTH", np.float32), + ("ALF", np.float32), + ("DEL", np.float32), + ("SPHLM3", np.float32), + ("SPHLM5", np.float32), + ("RP", np.float32), + ("GM", np.float32), + ("RAD", np.float32), + ("EXTNT1", np.float32), + ("EXTNT2", np.float32), + ("EXTNT3", np.float32), + ("MOID", np.float32), + ("ALBEDO", np.float32), + ("RMSW", np.float32), + ("RMSU", np.float32), + ("RMSN", np.float32), + ("RMSNT", np.float32), + ("RMSMT", np.float32), + ("RMSMN", np.float32), + ("EQUNOX", "|S4"), + ("PENAM", "|S6"), + ("SBNAM", "|S12"), + ("DARC", "|S9"), + ("COMNT3", "|S49"), + ("COMNT2", "|S80"), + ("DESIG", "|S13"), + ("COMEST", "|S14"), + ("IREF", "|S10"), + ("COMNAM", "|S29"), + ] + ) + ASTROQUERY_LOCAL_PATH = os.path.join(os.path.expanduser("~"), ".astroquery") + DBS_LOCAL_PATH = os.path.join(ASTROQUERY_LOCAL_PATH, "dastcom5", "dat") + AST_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") + COM_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") + FTP_DB_URL = "ftp://ssd.jpl.nasa.gov/pub/ssd/" + + +conf = Conf() + +from .core import * + +__all__ = [Dastcom5, Dastcom5Class] From 06c422e33d06b90d9e65f77a7e1546751af8e6f8 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 17 Jan 2019 21:02:40 +0530 Subject: [PATCH 16/28] Add the core package for querying DASTCOM5 --- astroquery/dastcom5/core.py | 188 ++++++++++ astroquery/dastcom5/dastcom5.py | 591 -------------------------------- 2 files changed, 188 insertions(+), 591 deletions(-) create mode 100755 astroquery/dastcom5/core.py delete mode 100644 astroquery/dastcom5/dastcom5.py diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py new file mode 100755 index 0000000000..38786c4f9a --- /dev/null +++ b/astroquery/dastcom5/core.py @@ -0,0 +1,188 @@ +import re +import os +import urllib.request +import zipfile + +from astropy.time import Time +from astropy.table import Table + +from . import conf +from ..query import BaseQuery +from ..utils import async_to_sync + +@async_to_sync +class Dastcom5Class(BaseQuery): + """ + Class for querying NEOs orbit from DASTCOM5. + """ + + def __init__(self, spk_id=None, name=None): + """ + Parameters: + ---------- + api_key : str + NASA OPEN APIs key (default: `DEMO_KEY`) + """ + super(NeowsClass, self).__init__() + self.spk_id = spk_id + self.name = name + self.local_path = conf.ASTROQUERY_LOCAL_PATH + self.ast_path = conf.AST_DB_PATH + self.com_path = conf.COM_DB_PATH + self.com_dtype = conf.COM_DTYPE + self.ast_dtype = conf.AST_DB_PATH + + + def download_dastcom5(self): + """Downloads DASTCOM5 database. + + Downloads and unzip DASTCOM5 file in default poliastro path (~/.poliastro). + + """ + + dastcom5_dir = os.path.join(self.local_path, "dastcom5") + dastcom5_zip_path = os.path.join(self.local_path, "dastcom5.zip") + + if os.path.isdir(dastcom5_dir): + raise FileExistsError( + "dastcom5 is already created in " + os.path.abspath(dastcom5_dir) + ) + if not zipfile.is_zipfile(dastcom5_zip_path): + if not os.path.isdir(self.local_path): + os.makedirs(self.local_path) + + urllib.request.urlretrieve( + FTP_DB_URL + "dastcom5.zip", dastcom5_zip_path, _show_download_progress + ) + with zipfile.ZipFile(dastcom5_zip_path) as myzip: + myzip.extractall(self.local_path) + + + def asteroid_db(): + """Return complete DASTCOM5 asteroid database. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + with open(AST_DB_PATH, "rb") as f: + f.seek(835, os.SEEK_SET) + data = np.fromfile(f, dtype=AST_DTYPE) + return data + + + + def comet_db(): + """Return complete DASTCOM5 comet database. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + with open(COM_DB_PATH, "rb") as f: + f.seek(976, os.SEEK_SET) + data = np.fromfile(f, dtype=COM_DTYPE) + return data + + + def _spk_id_from_name(self, name, cache=True): + payload = { + "sstr": name, + "orb": "0", + "log": "0", + "old": "0", + "cov": "0", + "cad": "0", + } + SBDB_URL = conf.sbdb_server + response = self._request( + "GET", SBDB_URL, params=payload, timeout=self.TIMEOUT, cache=cahe) + + response.raise_for_status() + soup = BeautifulSoup(response.text, "html.parser") + + # page_identifier is used to check what type of response page we are working with. + page_identifier = soup.find(attrs={"name": "top"}) + + # If there is a 'table' sibling, the object was found. + if page_identifier.find_next_sibling("table") is not None: + data = page_identifier.find_next_sibling( + "table").table.find_all("td") + + complete_string = "" + for string in data[1].stripped_strings: + complete_string += string + " " + match = re.compile(r"Classification: ([\S\s]+) SPK-ID: (\d+)").match( + complete_string + ) + if match: + self.spk_id = match.group(2) + return match.group(2) + + # If there is a 'center' sibling, it is a page with a list of possible objects + elif page_identifier.find_next_sibling("center") is not None: + object_list = page_identifier.find_next_sibling("center").table.find_all( + "td" + ) + bodies = "" + obj_num = min(len(object_list), 3) + for body in object_list[:obj_num]: + bodies += body.string + "\n" + raise ValueError( + str(len(object_list)) + " different bodies found:\n" + bodies + ) + + # If everything else failed + raise ValueError("Object could not be found. You can visit: " + + SBDB_URL + "?sstr=" + name + " for more information.") + + def from_spk_id(self, spk_id, cache=True): + """Return :py:class:`~astropy.table.Table` given a SPK-ID. + + Retrieve info from NASA NeoWS API, and therefore + it only works with NEAs (Near Earth Asteroids). + + Parameters + ---------- + spk_id : str + SPK-ID number, which is given to each body by JPL. + + Returns + ------- + Table : ~astropy.table.Table + NEA orbit parameters. + + """ + payload = {"api_key": self.api_key or DEFAULT_API_KEY} + + NEOWS_URL = conf.neows_server + response = self._request( + "GET", NEOWS_URL + spk_id, params=payload, timeout=self.TIMEOUT, cache=cache + ) + response.raise_for_status() + + orbital_data = response.json()["orbital_data"] + self.name = response.json()["name"] + abs_magnitude = response.json()["absolute_magnitude_h"] + column1 = ["name", "absolute_magnitude_h"] + column2 = [self.name, abs_magnitude] + column1.extend(orbital_data.keys()) + column2.extend(orbital_data.values()) + data = Table([column1, column2], names=("Parameters", "Values")) + self.epoch = Time( + float(orbital_data["epoch_osculation"]), format="jd", scale="tdb" + ) + + return data + + def from_name(self, name, cache=True): + if not self.spk_id: + self._spk_id_from_name(name=name, cache=cache) + return self.from_spk_id(spk_id=self.spk_id, cache=cache) + + +Neows = NeowsClass() diff --git a/astroquery/dastcom5/dastcom5.py b/astroquery/dastcom5/dastcom5.py deleted file mode 100644 index 6baa7ee394..0000000000 --- a/astroquery/dastcom5/dastcom5.py +++ /dev/null @@ -1,591 +0,0 @@ -"""NEOs orbit from DASTCOM5 database. - -""" -import os -import re -import urllib.request -import zipfile - -import astropy.units as u -import numpy as np -import pandas as pd -from astropy.time import Time - -from poliastro.bodies import Sun -from poliastro.frames import HeliocentricEclipticJ2000 -from poliastro.twobody.angles import M_to_nu -from poliastro.twobody.orbit import Orbit - -AST_DTYPE = np.dtype( - [ - ("NO", np.int32), - ("NOBS", np.int32), - ("OBSFRST", np.int32), - ("OBSLAST", np.int32), - ("EPOCH", np.float64), - ("CALEPO", np.float64), - ("MA", np.float64), - ("W", np.float64), - ("OM", np.float64), - ("IN", np.float64), - ("EC", np.float64), - ("A", np.float64), - ("QR", np.float64), - ("TP", np.float64), - ("TPCAL", np.float64), - ("TPFRAC", np.float64), - ("SOLDAT", np.float64), - ("SRC1", np.float64), - ("SRC2", np.float64), - ("SRC3", np.float64), - ("SRC4", np.float64), - ("SRC5", np.float64), - ("SRC6", np.float64), - ("SRC7", np.float64), - ("SRC8", np.float64), - ("SRC9", np.float64), - ("SRC10", np.float64), - ("SRC11", np.float64), - ("SRC12", np.float64), - ("SRC13", np.float64), - ("SRC14", np.float64), - ("SRC15", np.float64), - ("SRC16", np.float64), - ("SRC17", np.float64), - ("SRC18", np.float64), - ("SRC19", np.float64), - ("SRC20", np.float64), - ("SRC21", np.float64), - ("SRC22", np.float64), - ("SRC23", np.float64), - ("SRC24", np.float64), - ("SRC25", np.float64), - ("SRC26", np.float64), - ("SRC27", np.float64), - ("SRC28", np.float64), - ("SRC29", np.float64), - ("SRC30", np.float64), - ("SRC31", np.float64), - ("SRC32", np.float64), - ("SRC33", np.float64), - ("SRC34", np.float64), - ("SRC35", np.float64), - ("SRC36", np.float64), - ("SRC37", np.float64), - ("SRC38", np.float64), - ("SRC39", np.float64), - ("SRC40", np.float64), - ("SRC41", np.float64), - ("SRC42", np.float64), - ("SRC43", np.float64), - ("SRC44", np.float64), - ("SRC45", np.float64), - ("PRELTV", np.int8), - ("SPHMX3", np.int8), - ("SPHMX5", np.int8), - ("JGSEP", np.int8), - ("TWOBOD", np.int8), - ("NSATS", np.int8), - ("UPARM", np.int8), - ("LSRC", np.int8), - ("NDEL", np.int16), - ("NDOP", np.int16), - ("H", np.float32), - ("G", np.float32), - ("A1", np.float32), - ("A2", np.float32), - ("A3", np.float32), - ("R0", np.float32), - ("ALN", np.float32), - ("NM", np.float32), - ("NN", np.float32), - ("NK", np.float32), - ("LGK", np.float32), - ("RHO", np.float32), - ("AMRAT", np.float32), - ("ALF", np.float32), - ("DEL", np.float32), - ("SPHLM3", np.float32), - ("SPHLM5", np.float32), - ("RP", np.float32), - ("GM", np.float32), - ("RAD", np.float32), - ("EXTNT1", np.float32), - ("EXTNT2", np.float32), - ("EXTNT3", np.float32), - ("MOID", np.float32), - ("ALBEDO", np.float32), - ("BVCI", np.float32), - ("UBCI", np.float32), - ("IRCI", np.float32), - ("RMSW", np.float32), - ("RMSU", np.float32), - ("RMSN", np.float32), - ("RMSNT", np.float32), - ("RMSH", np.float32), - ("EQUNOX", "|S4"), - ("PENAM", "|S6"), - ("SBNAM", "|S12"), - ("SPTYPT", "|S5"), - ("SPTYPS", "|S5"), - ("DARC", "|S9"), - ("COMNT1", "|S41"), - ("COMNT2", "|S80"), - ("DESIG", "|S13"), - ("ASTEST", "|S8"), - ("IREF", "|S10"), - ("ASTNAM", "|S18"), - ] -) - -COM_DTYPE = np.dtype( - [ - ("NO", np.int32), - ("NOBS", np.int32), - ("OBSFRST", np.int32), - ("OBSLAST", np.int32), - ("EPOCH", np.float64), - ("CALEPO", np.float64), - ("MA", np.float64), - ("W", np.float64), - ("OM", np.float64), - ("IN", np.float64), - ("EC", np.float64), - ("A", np.float64), - ("QR", np.float64), - ("TP", np.float64), - ("TPCAL", np.float64), - ("TPFRAC", np.float64), - ("SOLDAT", np.float64), - ("SRC1", np.float64), - ("SRC2", np.float64), - ("SRC3", np.float64), - ("SRC4", np.float64), - ("SRC5", np.float64), - ("SRC6", np.float64), - ("SRC7", np.float64), - ("SRC8", np.float64), - ("SRC9", np.float64), - ("SRC10", np.float64), - ("SRC11", np.float64), - ("SRC12", np.float64), - ("SRC13", np.float64), - ("SRC14", np.float64), - ("SRC15", np.float64), - ("SRC16", np.float64), - ("SRC17", np.float64), - ("SRC18", np.float64), - ("SRC19", np.float64), - ("SRC20", np.float64), - ("SRC21", np.float64), - ("SRC22", np.float64), - ("SRC23", np.float64), - ("SRC24", np.float64), - ("SRC25", np.float64), - ("SRC26", np.float64), - ("SRC27", np.float64), - ("SRC28", np.float64), - ("SRC29", np.float64), - ("SRC30", np.float64), - ("SRC31", np.float64), - ("SRC32", np.float64), - ("SRC33", np.float64), - ("SRC34", np.float64), - ("SRC35", np.float64), - ("SRC36", np.float64), - ("SRC37", np.float64), - ("SRC38", np.float64), - ("SRC39", np.float64), - ("SRC40", np.float64), - ("SRC41", np.float64), - ("SRC42", np.float64), - ("SRC43", np.float64), - ("SRC44", np.float64), - ("SRC45", np.float64), - ("SRC46", np.float64), - ("SRC47", np.float64), - ("SRC48", np.float64), - ("SRC49", np.float64), - ("SRC50", np.float64), - ("SRC51", np.float64), - ("SRC52", np.float64), - ("SRC53", np.float64), - ("SRC54", np.float64), - ("SRC55", np.float64), - ("PRELTV", np.int8), - ("SPHMX3", np.int8), - ("SPHMX5", np.int8), - ("JGSEP", np.int8), - ("TWOBOD", np.int8), - ("NSATS", np.int8), - ("UPARM", np.int8), - ("LSRC", np.int8), - ("IPYR", np.int16), - ("NDEL", np.int16), - ("NDOP", np.int16), - ("NOBSMT", np.int16), - ("NOBSMN", np.int16), - ("H", np.float32), - ("G", np.float32), - ("M1 (MT)", np.float32), - ("M2 (MN)", np.float32), - ("K1 (MTSMT)", np.float32), - ("K2 (MNSMT)", np.float32), - ("PHCOF (MNP)", np.float32), - ("A1", np.float32), - ("A2", np.float32), - ("A3", np.float32), - ("DT", np.float32), - ("R0", np.float32), - ("ALN", np.float32), - ("NM", np.float32), - ("NN", np.float32), - ("NK", np.float32), - ("S0", np.float32), - ("TCL", np.float32), - ("RHO", np.float32), - ("AMRAT", np.float32), - ("AJ1", np.float32), - ("AJ2", np.float32), - ("ET1", np.float32), - ("ET2", np.float32), - ("DTH", np.float32), - ("ALF", np.float32), - ("DEL", np.float32), - ("SPHLM3", np.float32), - ("SPHLM5", np.float32), - ("RP", np.float32), - ("GM", np.float32), - ("RAD", np.float32), - ("EXTNT1", np.float32), - ("EXTNT2", np.float32), - ("EXTNT3", np.float32), - ("MOID", np.float32), - ("ALBEDO", np.float32), - ("RMSW", np.float32), - ("RMSU", np.float32), - ("RMSN", np.float32), - ("RMSNT", np.float32), - ("RMSMT", np.float32), - ("RMSMN", np.float32), - ("EQUNOX", "|S4"), - ("PENAM", "|S6"), - ("SBNAM", "|S12"), - ("DARC", "|S9"), - ("COMNT3", "|S49"), - ("COMNT2", "|S80"), - ("DESIG", "|S13"), - ("COMEST", "|S14"), - ("IREF", "|S10"), - ("COMNAM", "|S29"), - ] -) - -POLIASTRO_LOCAL_PATH = os.path.join(os.path.expanduser("~"), ".poliastro") -DBS_LOCAL_PATH = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5", "dat") -AST_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") -COM_DB_PATH = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") - -FTP_DB_URL = "ftp://ssd.jpl.nasa.gov/pub/ssd/" - - -def asteroid_db(): - """Return complete DASTCOM5 asteroid database. - - Returns - ------- - database : numpy.ndarray - Database with custom dtype. - - """ - with open(AST_DB_PATH, "rb") as f: - f.seek(835, os.SEEK_SET) - data = np.fromfile(f, dtype=AST_DTYPE) - return data - - - -def comet_db(): - """Return complete DASTCOM5 comet database. - - Returns - ------- - database : numpy.ndarray - Database with custom dtype. - - """ - with open(COM_DB_PATH, "rb") as f: - f.seek(976, os.SEEK_SET) - data = np.fromfile(f, dtype=COM_DTYPE) - return data - - -def orbit_from_name(name): - """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a name. - - Retrieve info from JPL DASTCOM5 database. - - Parameters - ---------- - name : str - NEO name. - - Returns - ------- - orbit : list (~poliastro.twobody.orbit.Orbit) - NEO orbits. - - """ - records = record_from_name(name) - orbits = [] - for record in records: - orbits.append(orbit_from_record(record)) - return orbits - - -def orbit_from_record(record): - """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a record. - - Retrieve info from JPL DASTCOM5 database. - - Parameters - ---------- - record : int - Object record. - - Returns - ------- - orbit : ~poliastro.twobody.orbit.Orbit - NEO orbit. - - """ - body_data = read_record(record) - a = body_data["A"].item() * u.au - ecc = body_data["EC"].item() * u.one - inc = body_data["IN"].item() * u.deg - raan = body_data["OM"].item() * u.deg - argp = body_data["W"].item() * u.deg - m = body_data["MA"].item() * u.deg - nu = M_to_nu(m, ecc) - epoch = Time(body_data["EPOCH"].item(), format="jd", scale="tdb") - - orbit = Orbit.from_classical(Sun, a, ecc, inc, raan, argp, nu, epoch) - orbit._frame = HeliocentricEclipticJ2000(obstime=epoch) - return orbit - - -def record_from_name(name): - """Search `dastcom.idx` and return logical records that match a given string. - - Body name, SPK-ID, or alternative designations can be used. - - Parameters - ---------- - name : str - Body name. - - Returns - ------- - records : list (int) - DASTCOM5 database logical records matching str. - - """ - records = [] - lines = string_record_from_name(name) - for line in lines: - records.append(int(line[:6].lstrip())) - return records - - -def string_record_from_name(name): - """Search `dastcom.idx` and return body full record. - - Search DASTCOM5 index and return body records that match string, - containing logical record, name, alternative designations, SPK-ID, etc. - - Parameters - ---------- - name : str - Body name. - - Returns - ------- - lines: list(str) - Body records - """ - - idx_path = os.path.join(DBS_LOCAL_PATH, "dastcom.idx") - lines = [] - with open(idx_path, "r") as inF: - for line in inF: - if re.search(r"\b" + name.casefold() + r"\b", line.casefold()): - lines.append(line) - return lines - - -def read_headers(): - """Read `DASTCOM5` headers and return asteroid and comet headers. - - Headers are two numpy arrays with custom dtype. - - Returns - ------- - ast_header, com_header : tuple (numpy.ndarray) - DASTCOM5 headers. - - """ - - ast_path = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") - ast_dtype = np.dtype( - [ - ("IBIAS1", np.int32), - ("BEGINP1", "|S8"), - ("BEGINP2", "|S8"), - ("BEGINP3", "|S8"), - ("ENDPT1", "|S8"), - ("ENDPT2", "|S8"), - ("ENDPT3", "|S8"), - ("CALDATE", "|S19"), - ("JDDATE", np.float64), - ("FTYP", "|S1"), - ("BYTE2A", np.int16), - ("IBIAS0", np.int32), - ] - ) - - with open(ast_path, "rb") as f: - ast_header = np.fromfile(f, dtype=ast_dtype, count=1) - - com_path = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") - com_dtype = np.dtype( - [ - ("IBIAS2", np.int32), - ("BEGINP1", "|S8"), - ("BEGINP2", "|S8"), - ("BEGINP3", "|S8"), - ("ENDPT1", "|S8"), - ("ENDPT2", "|S8"), - ("ENDPT3", "|S8"), - ("CALDATE", "|S19"), - ("JDDATE", np.float64), - ("FTYP", "|S1"), - ("BYTE2C", np.int16), - ] - ) - - with open(com_path, "rb") as f: - com_header = np.fromfile(f, dtype=com_dtype, count=1) - - return ast_header, com_header - - -def read_record(record): - """Read `DASTCOM5` record and return body data. - - Body data consists of numpy array with custom dtype. - - Parameters - ---------- - record : int - Body record. - - Returns - ------- - body_data : numpy.ndarray - Body information. - - """ - ast_header, com_header = read_headers() - ast_path = os.path.join(DBS_LOCAL_PATH, "dast5_le.dat") - com_path = os.path.join(DBS_LOCAL_PATH, "dcom5_le.dat") - # ENDPT1 indicates end of numbered asteroids records - if record <= int(ast_header["ENDPT2"][0].item()): - # ENDPT2 indicates end of unnumbered asteroids records - if record <= int(ast_header["ENDPT1"][0].item()): - # phis_rec = record_size * (record_number - IBIAS - 1 (header record)) - phis_rec = 835 * (record - ast_header["IBIAS0"][0].item() - 1) - else: - phis_rec = 835 * (record - ast_header["IBIAS1"][0].item() - 1) - - with open(ast_path, "rb") as f: - f.seek(phis_rec, os.SEEK_SET) - body_data = np.fromfile(f, dtype=AST_DTYPE, count=1) - else: - phis_rec = 976 * (record - com_header["IBIAS2"][0].item() - 1) - with open(com_path, "rb") as f: - f.seek(phis_rec, os.SEEK_SET) - body_data = np.fromfile(f, dtype=COM_DTYPE, count=1) - return body_data - - -def download_dastcom5(): - """Downloads DASTCOM5 database. - - Downloads and unzip DASTCOM5 file in default poliastro path (~/.poliastro). - - """ - - dastcom5_dir = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5") - dastcom5_zip_path = os.path.join(POLIASTRO_LOCAL_PATH, "dastcom5.zip") - - if os.path.isdir(dastcom5_dir): - raise FileExistsError( - "dastcom5 is already created in " + os.path.abspath(dastcom5_dir) - ) - if not zipfile.is_zipfile(dastcom5_zip_path): - if not os.path.isdir(POLIASTRO_LOCAL_PATH): - os.makedirs(POLIASTRO_LOCAL_PATH) - - urllib.request.urlretrieve( - FTP_DB_URL + "dastcom5.zip", dastcom5_zip_path, _show_download_progress - ) - with zipfile.ZipFile(dastcom5_zip_path) as myzip: - myzip.extractall(POLIASTRO_LOCAL_PATH) - - -def _show_download_progress(transferred, block, totalsize): - trans_mb = transferred * block / (1024 * 1024) - total_mb = totalsize / (1024 * 1024) - print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) - - -def entire_db(): - """Return complete DASTCOM5 database. - - Merge asteroid and comet databases, only with fields - related to orbital data, discarding the rest. - - Returns - ------- - database : numpy.ndarray - Database with custom dtype. - - """ - ast_database = asteroid_db() - com_database = comet_db() - - ast_database = pd.DataFrame( - ast_database[ - list(ast_database.dtype.names[:17]) - + list(ast_database.dtype.names[-4:-3]) - + list(ast_database.dtype.names[-2:]) - ] - ) - ast_database.rename( - columns={"ASTNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True - ) - com_database = pd.DataFrame( - com_database[ - list(com_database.dtype.names[:17]) - + list(com_database.dtype.names[-4:-3]) - + list(com_database.dtype.names[-2:]) - ] - ) - com_database.rename( - columns={"COMNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True - ) - df = ast_database.append(com_database, ignore_index=True) - df[["NAME", "DESIG", "IREF"]] = df[["NAME", "DESIG", "IREF"]].apply( - lambda x: x.str.strip().str.decode("utf-8") - ) - return df From 6c8cd85bda4f9d844fe0a4cd2de9119c6b5e1490 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 17 Jan 2019 22:16:34 +0530 Subject: [PATCH 17/28] Create a stack of tables for returning multiple orbits --- astroquery/dastcom5/__init__.py | 3 + astroquery/dastcom5/core.py | 332 +++++++++++++++++++++++--------- 2 files changed, 245 insertions(+), 90 deletions(-) diff --git a/astroquery/dastcom5/__init__.py b/astroquery/dastcom5/__init__.py index 105b4e52fe..6c66fcbbc1 100755 --- a/astroquery/dastcom5/__init__.py +++ b/astroquery/dastcom5/__init__.py @@ -11,6 +11,9 @@ All of the methods are coded as part of SOCIS 2017 for poliastro by Antonio Hidalgo[1]. """ +import os +import numpy as np + from astropy import config as _config diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 38786c4f9a..b6e90f3916 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -2,9 +2,11 @@ import os import urllib.request import zipfile +import numpy as np from astropy.time import Time -from astropy.table import Table +from astropy.table import Table, vstack +import astropy.units as u from . import conf from ..query import BaseQuery @@ -23,14 +25,16 @@ def __init__(self, spk_id=None, name=None): api_key : str NASA OPEN APIs key (default: `DEMO_KEY`) """ - super(NeowsClass, self).__init__() + super(Dastcom5Class, self).__init__() self.spk_id = spk_id self.name = name self.local_path = conf.ASTROQUERY_LOCAL_PATH + self.dbs_path = conf.DBS_LOCAL_PATH self.ast_path = conf.AST_DB_PATH self.com_path = conf.COM_DB_PATH self.com_dtype = conf.COM_DTYPE - self.ast_dtype = conf.AST_DB_PATH + self.ast_dtype = conf.AST_DTYPE + self.ftp_url = conf.FTP_DB_URL def download_dastcom5(self): @@ -52,13 +56,18 @@ def download_dastcom5(self): os.makedirs(self.local_path) urllib.request.urlretrieve( - FTP_DB_URL + "dastcom5.zip", dastcom5_zip_path, _show_download_progress + self.ftp_url + "dastcom5.zip", dastcom5_zip_path, self._show_download_progress ) with zipfile.ZipFile(dastcom5_zip_path) as myzip: myzip.extractall(self.local_path) + def _show_download_progress(self, transferred, block, totalsize): + trans_mb = transferred * block / (1024 * 1024) + total_mb = totalsize / (1024 * 1024) + print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) - def asteroid_db(): + + def asteroid_db(self): """Return complete DASTCOM5 asteroid database. Returns @@ -67,14 +76,14 @@ def asteroid_db(): Database with custom dtype. """ - with open(AST_DB_PATH, "rb") as f: + with open(self.ast_path, "rb") as f: f.seek(835, os.SEEK_SET) - data = np.fromfile(f, dtype=AST_DTYPE) + data = np.fromfile(f, dtype=self.ast_dtype) return data - def comet_db(): + def comet_db(self): """Return complete DASTCOM5 comet database. Returns @@ -83,106 +92,249 @@ def comet_db(): Database with custom dtype. """ - with open(COM_DB_PATH, "rb") as f: + with open(self.com_path, "rb") as f: f.seek(976, os.SEEK_SET) - data = np.fromfile(f, dtype=COM_DTYPE) + data = np.fromfile(f, dtype=self.com_dtype) return data - def _spk_id_from_name(self, name, cache=True): - payload = { - "sstr": name, - "orb": "0", - "log": "0", - "old": "0", - "cov": "0", - "cad": "0", - } - SBDB_URL = conf.sbdb_server - response = self._request( - "GET", SBDB_URL, params=payload, timeout=self.TIMEOUT, cache=cahe) - - response.raise_for_status() - soup = BeautifulSoup(response.text, "html.parser") - - # page_identifier is used to check what type of response page we are working with. - page_identifier = soup.find(attrs={"name": "top"}) - - # If there is a 'table' sibling, the object was found. - if page_identifier.find_next_sibling("table") is not None: - data = page_identifier.find_next_sibling( - "table").table.find_all("td") - - complete_string = "" - for string in data[1].stripped_strings: - complete_string += string + " " - match = re.compile(r"Classification: ([\S\s]+) SPK-ID: (\d+)").match( - complete_string - ) - if match: - self.spk_id = match.group(2) - return match.group(2) - - # If there is a 'center' sibling, it is a page with a list of possible objects - elif page_identifier.find_next_sibling("center") is not None: - object_list = page_identifier.find_next_sibling("center").table.find_all( - "td" - ) - bodies = "" - obj_num = min(len(object_list), 3) - for body in object_list[:obj_num]: - bodies += body.string + "\n" - raise ValueError( - str(len(object_list)) + " different bodies found:\n" + bodies - ) + def read_headers(self): + """Read `DASTCOM5` headers and return asteroid and comet headers. + + Headers are two numpy arrays with custom dtype. + + Returns + ------- + ast_header, com_header : tuple (numpy.ndarray) + DASTCOM5 headers. + + """ + + ast_path = os.path.join(self.dbs_path, "dast5_le.dat") + ast_dtype = np.dtype( + [ + ("IBIAS1", np.int32), + ("BEGINP1", "|S8"), + ("BEGINP2", "|S8"), + ("BEGINP3", "|S8"), + ("ENDPT1", "|S8"), + ("ENDPT2", "|S8"), + ("ENDPT3", "|S8"), + ("CALDATE", "|S19"), + ("JDDATE", np.float64), + ("FTYP", "|S1"), + ("BYTE2A", np.int16), + ("IBIAS0", np.int32), + ] + ) + + with open(ast_path, "rb") as f: + ast_header = np.fromfile(f, dtype=ast_dtype, count=1) + + com_path = os.path.join(self.dbs_path, "dcom5_le.dat") + com_dtype = np.dtype( + [ + ("IBIAS2", np.int32), + ("BEGINP1", "|S8"), + ("BEGINP2", "|S8"), + ("BEGINP3", "|S8"), + ("ENDPT1", "|S8"), + ("ENDPT2", "|S8"), + ("ENDPT3", "|S8"), + ("CALDATE", "|S19"), + ("JDDATE", np.float64), + ("FTYP", "|S1"), + ("BYTE2C", np.int16), + ] + ) + + with open(com_path, "rb") as f: + com_header = np.fromfile(f, dtype=com_dtype, count=1) + + return ast_header, com_header - # If everything else failed - raise ValueError("Object could not be found. You can visit: " + - SBDB_URL + "?sstr=" + name + " for more information.") - def from_spk_id(self, spk_id, cache=True): - """Return :py:class:`~astropy.table.Table` given a SPK-ID. + def entire_db(self): + """Return complete DASTCOM5 database. - Retrieve info from NASA NeoWS API, and therefore - it only works with NEAs (Near Earth Asteroids). + Merge asteroid and comet databases, only with fields + related to orbital data, discarding the rest. + + Returns + ------- + database : numpy.ndarray + Database with custom dtype. + + """ + ast_database = self.asteroid_db() + com_database = self.comet_db() + + ast_database = pd.DataFrame( + ast_database[ + list(ast_database.dtype.names[:17]) + + list(ast_database.dtype.names[-4:-3]) + + list(ast_database.dtype.names[-2:]) + ] + ) + ast_database.rename( + columns={"ASTNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True + ) + com_database = pd.DataFrame( + com_database[ + list(com_database.dtype.names[:17]) + + list(com_database.dtype.names[-4:-3]) + + list(com_database.dtype.names[-2:]) + ] + ) + com_database.rename( + columns={"COMNAM": "NAME", "NO": "NUMBER", "CALEPO": "CALEPOCH"}, inplace=True + ) + df = ast_database.append(com_database, ignore_index=True) + df[["NAME", "DESIG", "IREF"]] = df[["NAME", "DESIG", "IREF"]].apply( + lambda x: x.str.strip().str.decode("utf-8") + ) + return df + + + def read_record(self, record): + """Read `DASTCOM5` record and return body data. + + Body data consists of numpy array with custom dtype. Parameters ---------- - spk_id : str - SPK-ID number, which is given to each body by JPL. + record : int + Body record. Returns ------- - Table : ~astropy.table.Table - NEA orbit parameters. + body_data : numpy.ndarray + Body information. """ - payload = {"api_key": self.api_key or DEFAULT_API_KEY} + ast_header, com_header = self.read_headers() + ast_path = os.path.join(self.dbs_path, "dast5_le.dat") + com_path = os.path.join(self.dbs_path, "dcom5_le.dat") + # ENDPT1 indicates end of numbered asteroids records + if record <= int(ast_header["ENDPT2"][0].item()): + # ENDPT2 indicates end of unnumbered asteroids records + if record <= int(ast_header["ENDPT1"][0].item()): + # phis_rec = record_size * (record_number - IBIAS - 1 (header record)) + phis_rec = 835 * (record - ast_header["IBIAS0"][0].item() - 1) + else: + phis_rec = 835 * (record - ast_header["IBIAS1"][0].item() - 1) + + with open(ast_path, "rb") as f: + f.seek(phis_rec, os.SEEK_SET) + body_data = np.fromfile(f, dtype=self.ast_dtype, count=1) + else: + phis_rec = 976 * (record - com_header["IBIAS2"][0].item() - 1) + with open(com_path, "rb") as f: + f.seek(phis_rec, os.SEEK_SET) + body_data = np.fromfile(f, dtype=self.com_dtype, count=1) + return body_data + + def orbit_from_name(self, name): + """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a name. + + Retrieve info from JPL DASTCOM5 database. - NEOWS_URL = conf.neows_server - response = self._request( - "GET", NEOWS_URL + spk_id, params=payload, timeout=self.TIMEOUT, cache=cache - ) - response.raise_for_status() - - orbital_data = response.json()["orbital_data"] - self.name = response.json()["name"] - abs_magnitude = response.json()["absolute_magnitude_h"] - column1 = ["name", "absolute_magnitude_h"] - column2 = [self.name, abs_magnitude] - column1.extend(orbital_data.keys()) - column2.extend(orbital_data.values()) - data = Table([column1, column2], names=("Parameters", "Values")) - self.epoch = Time( - float(orbital_data["epoch_osculation"]), format="jd", scale="tdb" - ) + Parameters + ---------- + name : str + NEO name. + + Returns + ------- + Table : ~astropy.table.Table + Near Earth Asteroid/Comet orbit parameters, all stacked. + + """ + records = self.record_from_name(name) + table = self.orbit_from_record(records[0]) + for i in range(1, len(records)): + table = vstack([table, self.orbit_from_record(records[i])[1]]) + return table + + + def orbit_from_record(self, record): + """Return :py:class:`~astropy.table.Table` given a record. + + Retrieve info from JPL DASTCOM5 database. + + Parameters + ---------- + record : int + Object record. + + Returns + ------- + Table : ~astropy.table.Table + Near Earth Asteroid/Comet orbit parameters. + + """ + body_data = self.read_record(record) + a = body_data["A"].item() + ecc = body_data["EC"].item() + inc = body_data["IN"].item() + raan = body_data["OM"].item() + argp = body_data["W"].item() + m = body_data["MA"].item() + epoch = Time(body_data["EPOCH"].item(), format="jd", scale="tdb") + column2 = (record, a, ecc, inc, raan, argp, m, epoch) + column1 = ("record", "a", "ecc", "inc", "raan", "argp", "m", "EPOCH") + data = Table(rows=[column1, column2]) return data - def from_name(self, name, cache=True): - if not self.spk_id: - self._spk_id_from_name(name=name, cache=cache) - return self.from_spk_id(spk_id=self.spk_id, cache=cache) + + def record_from_name(self, name): + """Search `dastcom.idx` and return logical records that match a given string. + + Body name, SPK-ID, or alternative designations can be used. + + Parameters + ---------- + name : str + Body name. + + Returns + ------- + records : list (int) + DASTCOM5 database logical records matching str. + + """ + records = [] + lines = self.string_record_from_name(name) + for line in lines: + records.append(int(line[:6].lstrip())) + return records + + + def string_record_from_name(self, name): + """Search `dastcom.idx` and return body full record. + + Search DASTCOM5 index and return body records that match string, + containing logical record, name, alternative designations, SPK-ID, etc. + + Parameters + ---------- + name : str + Body name. + + Returns + ------- + lines: list(str) + Body records + """ + + idx_path = os.path.join(self.dbs_path, "dastcom.idx") + lines = [] + with open(idx_path, "r") as inF: + for line in inF: + if re.search(r"\b" + name.casefold() + r"\b", line.casefold()): + lines.append(line) + return lines -Neows = NeowsClass() +Dastcom5 = Dastcom5Class() From 8739e0a7acc8716a9c7d16158cdd952115e7ffb2 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 17 Jan 2019 22:55:18 +0530 Subject: [PATCH 18/28] Complete the DASTCOM5 Module, Add tests --- astroquery/dastcom5/core.py | 18 +--- astroquery/dastcom5/tests/__init__.py | 0 astroquery/dastcom5/tests/test_dastcom5.py | 106 +++++++++++++++++++++ 3 files changed, 111 insertions(+), 13 deletions(-) create mode 100644 astroquery/dastcom5/tests/__init__.py create mode 100644 astroquery/dastcom5/tests/test_dastcom5.py diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index b6e90f3916..0b5c2f0ab8 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -12,6 +12,7 @@ from ..query import BaseQuery from ..utils import async_to_sync + @async_to_sync class Dastcom5Class(BaseQuery): """ @@ -36,11 +37,10 @@ def __init__(self, spk_id=None, name=None): self.ast_dtype = conf.AST_DTYPE self.ftp_url = conf.FTP_DB_URL - def download_dastcom5(self): """Downloads DASTCOM5 database. - Downloads and unzip DASTCOM5 file in default poliastro path (~/.poliastro). + Downloads and unzip DASTCOM5 file in default astroquery path (~/.astroquery). """ @@ -49,7 +49,8 @@ def download_dastcom5(self): if os.path.isdir(dastcom5_dir): raise FileExistsError( - "dastcom5 is already created in " + os.path.abspath(dastcom5_dir) + "dastcom5 is already created in " + \ + os.path.abspath(dastcom5_dir) ) if not zipfile.is_zipfile(dastcom5_zip_path): if not os.path.isdir(self.local_path): @@ -66,7 +67,6 @@ def _show_download_progress(self, transferred, block, totalsize): total_mb = totalsize / (1024 * 1024) print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) - def asteroid_db(self): """Return complete DASTCOM5 asteroid database. @@ -81,8 +81,6 @@ def asteroid_db(self): data = np.fromfile(f, dtype=self.ast_dtype) return data - - def comet_db(self): """Return complete DASTCOM5 comet database. @@ -97,7 +95,6 @@ def comet_db(self): data = np.fromfile(f, dtype=self.com_dtype) return data - def read_headers(self): """Read `DASTCOM5` headers and return asteroid and comet headers. @@ -153,7 +150,6 @@ def read_headers(self): return ast_header, com_header - def entire_db(self): """Return complete DASTCOM5 database. @@ -195,7 +191,6 @@ def entire_db(self): ) return df - def read_record(self, record): """Read `DASTCOM5` record and return body data. @@ -235,7 +230,7 @@ def read_record(self, record): return body_data def orbit_from_name(self, name): - """Return :py:class:`~poliastro.twobody.orbit.Orbit` given a name. + """Return :py:class:`~astropy.table.Table` given a name. Retrieve info from JPL DASTCOM5 database. @@ -256,7 +251,6 @@ def orbit_from_name(self, name): table = vstack([table, self.orbit_from_record(records[i])[1]]) return table - def orbit_from_record(self, record): """Return :py:class:`~astropy.table.Table` given a record. @@ -287,7 +281,6 @@ def orbit_from_record(self, record): return data - def record_from_name(self, name): """Search `dastcom.idx` and return logical records that match a given string. @@ -310,7 +303,6 @@ def record_from_name(self, name): records.append(int(line[:6].lstrip())) return records - def string_record_from_name(self, name): """Search `dastcom.idx` and return body full record. diff --git a/astroquery/dastcom5/tests/__init__.py b/astroquery/dastcom5/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py new file mode 100644 index 0000000000..5151a89079 --- /dev/null +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -0,0 +1,106 @@ +import os +from unittest import mock + +import numpy as np +import pytest + +import ...dastcom5 +from ...dastcom5 import Dastcom5 + +from astropy.table import Table + + +@mock.patch("astroquery.dastcom5.np.fromfile") +@mock.patch("astroquery.dastcom5.open") +def test_asteroid_db_is_called_with_right_path(mock_open, mock_np_fromfile): + Dastcom5.asteroid_db() + mock_open.assert_called_with(Dastcom5.ast_path, "rb") + + +@mock.patch("astroquery.dastcom5.np.fromfile") +@mock.patch("astroquery.dastcom5.open") +def test_comet_db_is_called_with_right_path(mock_open, mock_np_fromfile): + Dastcom5.comet_db() + mock_open.assert_called_with(Dastcom5.com_path, "rb") + + +@mock.patch("astroquery.dastcom5.np.fromfile") +@mock.patch("astroquery.dastcom5.open") +def test_read_headers(mock_open, mock_np_fromfile): + Dastcom5.read_headers() + mock_open.assert_any_call( + os.path.join(Dastcom5.dbs_path, "dast5_le.dat"), "rb" + ) + mock_open.assert_any_call( + os.path.join(Dastcom5.dbs_path, "dcom5_le.dat"), "rb" + ) + + +@mock.patch("astroquery.dastcom5.Dastcom5.read_headers") +@mock.patch("astroquery.dastcom5.np.fromfile") +@mock.patch("astroquery.dastcom5.open") +def test_read_record(mock_open, mock_np_fromfile, mock_read_headers): + mocked_ast_headers = np.array( + [(3184, -1, b"00740473", b"00496815")], + dtype=[ + ("IBIAS1", np.int32), + ("IBIAS0", np.int32), + ("ENDPT2", "|S8"), + ("ENDPT1", "|S8"), + ], + ) + mocked_com_headers = np.array([(99999,)], dtype=[("IBIAS2", " Date: Thu, 17 Jan 2019 23:02:48 +0530 Subject: [PATCH 19/28] Fix PEP8 Errors --- astroquery/dastcom5/core.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 0b5c2f0ab8..2fa75413bb 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -49,8 +49,8 @@ def download_dastcom5(self): if os.path.isdir(dastcom5_dir): raise FileExistsError( - "dastcom5 is already created in " + \ - os.path.abspath(dastcom5_dir) + "dastcom5 is already created in " + + os.path.abspath(dastcom5_dir) ) if not zipfile.is_zipfile(dastcom5_zip_path): if not os.path.isdir(self.local_path): @@ -167,9 +167,9 @@ def entire_db(self): ast_database = pd.DataFrame( ast_database[ - list(ast_database.dtype.names[:17]) - + list(ast_database.dtype.names[-4:-3]) - + list(ast_database.dtype.names[-2:]) + list(ast_database.dtype.names[:17]) + + list(ast_database.dtype.names[-4:-3]) + + list(ast_database.dtype.names[-2:]) ] ) ast_database.rename( @@ -177,9 +177,9 @@ def entire_db(self): ) com_database = pd.DataFrame( com_database[ - list(com_database.dtype.names[:17]) - + list(com_database.dtype.names[-4:-3]) - + list(com_database.dtype.names[-2:]) + list(com_database.dtype.names[:17]) + + list(com_database.dtype.names[-4:-3]) + + list(com_database.dtype.names[-2:]) ] ) com_database.rename( From d3986f33e571160b33cd3a00d11af084afc604a6 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Mon, 28 Jan 2019 10:16:08 +0530 Subject: [PATCH 20/28] Minor changes --- astroquery/dastcom5/core.py | 1 + astroquery/dastcom5/tests/test_dastcom5.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 2fa75413bb..fedcef7036 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -3,6 +3,7 @@ import urllib.request import zipfile import numpy as np +import pandas as pd from astropy.time import Time from astropy.table import Table, vstack diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py index 5151a89079..d695a0ab52 100644 --- a/astroquery/dastcom5/tests/test_dastcom5.py +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -4,7 +4,6 @@ import numpy as np import pytest -import ...dastcom5 from ...dastcom5 import Dastcom5 from astropy.table import Table From bc0d1eb77cb6159c62f1f38ba37b838b0c31700e Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Mon, 28 Jan 2019 10:27:37 +0530 Subject: [PATCH 21/28] Add Python2 Support --- astroquery/dastcom5/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index fedcef7036..59cd0d0c1d 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -1,3 +1,4 @@ +from __future__ import print_function import re import os import urllib.request From 614b8359605331a118bba947f4b04959066266bf Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Wed, 30 Jan 2019 00:12:02 +0530 Subject: [PATCH 22/28] Work on suggestions --- astroquery/dastcom5/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 59cd0d0c1d..0977a448a7 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -268,7 +268,7 @@ def orbit_from_record(self, record): Table : ~astropy.table.Table Near Earth Asteroid/Comet orbit parameters. - """ + """ body_data = self.read_record(record) a = body_data["A"].item() ecc = body_data["EC"].item() @@ -320,8 +320,8 @@ def string_record_from_name(self, name): ------- lines: list(str) Body records - """ + """ idx_path = os.path.join(self.dbs_path, "dastcom.idx") lines = [] with open(idx_path, "r") as inF: From cf2859385cc10ec30aa3c048461477af149c5385 Mon Sep 17 00:00:00 2001 From: Ayush Suhane Date: Tue, 5 Feb 2019 22:25:17 +0530 Subject: [PATCH 23/28] Fix download progress issue --- astroquery/dastcom5/core.py | 20 ++++--- astroquery/dastcom5/tests/test_dastcom5.py | 66 ++++++++++------------ 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 0977a448a7..d795a80eca 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -5,6 +5,7 @@ import zipfile import numpy as np import pandas as pd +from tqdm import tqdm from astropy.time import Time from astropy.table import Table, vstack @@ -38,6 +39,7 @@ def __init__(self, spk_id=None, name=None): self.com_dtype = conf.COM_DTYPE self.ast_dtype = conf.AST_DTYPE self.ftp_url = conf.FTP_DB_URL + self._show_download_progress = self._Show_Download_Progress def download_dastcom5(self): """Downloads DASTCOM5 database. @@ -58,16 +60,20 @@ def download_dastcom5(self): if not os.path.isdir(self.local_path): os.makedirs(self.local_path) - urllib.request.urlretrieve( - self.ftp_url + "dastcom5.zip", dastcom5_zip_path, self._show_download_progress - ) + print("Downloading datscom5.zip") + with self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip") as t: + urllib.request.urlretrieve(self.ftp_url + "dastcom5.zip", filename=dastcom5_zip_path, reporthook=t.update_to) + with zipfile.ZipFile(dastcom5_zip_path) as myzip: myzip.extractall(self.local_path) - def _show_download_progress(self, transferred, block, totalsize): - trans_mb = transferred * block / (1024 * 1024) - total_mb = totalsize / (1024 * 1024) - print("%.2f MB / %.2f MB" % (trans_mb, total_mb), end="\r", flush=True) + class _Show_Download_Progress(tqdm): + """Helper class for displaying download progress bar when using download_dastcom5() function + """ + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) def asteroid_db(self): """Return complete DASTCOM5 asteroid database. diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py index d695a0ab52..36e163f883 100644 --- a/astroquery/dastcom5/tests/test_dastcom5.py +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -1,31 +1,31 @@ import os -from unittest import mock import numpy as np import pytest +from pytest_mock import mocker from ...dastcom5 import Dastcom5 from astropy.table import Table -@mock.patch("astroquery.dastcom5.np.fromfile") -@mock.patch("astroquery.dastcom5.open") -def test_asteroid_db_is_called_with_right_path(mock_open, mock_np_fromfile): +def test_asteroid_db_is_called_with_right_path(mocker): + mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") + mock_open = mocker.patch("poliastro.neos.dastcom5.open") Dastcom5.asteroid_db() mock_open.assert_called_with(Dastcom5.ast_path, "rb") -@mock.patch("astroquery.dastcom5.np.fromfile") -@mock.patch("astroquery.dastcom5.open") -def test_comet_db_is_called_with_right_path(mock_open, mock_np_fromfile): +def test_comet_db_is_called_with_right_path(mocker): + mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") + mock_open = mocker.patch("poliastro.neos.dastcom5.open") Dastcom5.comet_db() mock_open.assert_called_with(Dastcom5.com_path, "rb") -@mock.patch("astroquery.dastcom5.np.fromfile") -@mock.patch("astroquery.dastcom5.open") -def test_read_headers(mock_open, mock_np_fromfile): +def test_read_headers(mocker): + mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") + mock_open = mocker.patch("poliastro.neos.dastcom5.open") Dastcom5.read_headers() mock_open.assert_any_call( os.path.join(Dastcom5.dbs_path, "dast5_le.dat"), "rb" @@ -35,10 +35,10 @@ def test_read_headers(mock_open, mock_np_fromfile): ) -@mock.patch("astroquery.dastcom5.Dastcom5.read_headers") -@mock.patch("astroquery.dastcom5.np.fromfile") -@mock.patch("astroquery.dastcom5.open") -def test_read_record(mock_open, mock_np_fromfile, mock_read_headers): +def test_read_record(mocker): + mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") + mock_open = mocker.patch("poliastro.neos.dastcom5.open") + mock_read_headers = mocker.patch("astroquery.dastcom5.Dastcom5.read_headers") mocked_ast_headers = np.array( [(3184, -1, b"00740473", b"00496815")], dtype=[ @@ -61,13 +61,11 @@ def test_read_record(mock_open, mock_np_fromfile, mock_read_headers): ) -@mock.patch("astroquery.dastcom5.os.makedirs") -@mock.patch("astroquery.dastcom5.zipfile") -@mock.patch("astroquery.dastcom5.os.path.isdir") -@mock.patch("astroquery.dastcom5.urllib.request") -def test_download_dastcom5_raises_error_when_folder_exists( - mock_request, mock_isdir, mock_zipfile, mock_makedirs -): +def test_download_dastcom5_raises_error_when_folder_exists(mocker): + mock_request = mocker.patch("astroquery.dastcom5.urllib.request") + mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") + mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") + mock_makedirs = mocker.patch("astroquery.dastcom5.os.makedirs") mock_isdir.side_effect = lambda x: x == os.path.join( Dastcom5.local_path, "dastcom5" ) @@ -78,28 +76,26 @@ def test_download_dastcom5_raises_error_when_folder_exists( ) -@mock.patch("astroquery.dastcom5.urllib.request") -@mock.patch("astroquery.dastcom5.os.makedirs") -@mock.patch("astroquery.dastcom5.zipfile") -@mock.patch("astroquery.dastcom5.os.path.isdir") -def test_download_dastcom5_creates_folder( - mock_isdir, mock_zipfile, mock_makedirs, mock_request -): +def test_download_dastcom5_creates_folder(mocker): + mock_request = mocker.patch("astroquery.dastcom5.urllib.request") + mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") + mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") + mock_makedirs = mocker.patch("astroquery.dastcom5.os.makedirs") mock_isdir.return_value = False mock_zipfile.is_zipfile.return_value = False Dastcom5.download_dastcom5() mock_makedirs.assert_called_once_with(Dastcom5.local_path) -@mock.patch("astroquery.dastcom5.zipfile") -@mock.patch("astroquery.dastcom5.os.path.isdir") -@mock.patch("astroquery.dastcom5.urllib.request.urlretrieve") -def test_download_dastcom5_downloads_file(mock_request, mock_isdir, mock_zipfile): +def test_download_dastcom5_downloads_file(mocker): + mock_request = mocker.patch("astroquery.dastcom5.urllib.request.urlretrieve") + mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") + mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") mock_isdir.side_effect = lambda x: x == Dastcom5.local_path mock_zipfile.is_zipfile.return_value = False Dastcom5.download_dastcom5() mock_request.assert_called_once_with( - Dastcom5.FTP_DB_URL + "dastcom5.zip", - os.path.join(Dastcom5.local_path, "dastcom5.zip"), - Dastcom5._show_download_progress, + Dastcom5.ftp_url + "dastcom5.zip", + filename=os.path.join(Dastcom5.local_path, "dastcom5.zip"), + reporthook=Dastcom5.self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip").update_to, ) From af17efd52a402c868fbbc0427e5d5f0b8816ad60 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Tue, 5 Feb 2019 22:27:11 +0530 Subject: [PATCH 24/28] Remove vstack --- astroquery/dastcom5/core.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index d795a80eca..c9e886c170 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -8,7 +8,7 @@ from tqdm import tqdm from astropy.time import Time -from astropy.table import Table, vstack +from astropy.table import Table import astropy.units as u from . import conf @@ -254,10 +254,9 @@ def orbit_from_name(self, name): """ records = self.record_from_name(name) - table = self.orbit_from_record(records[0]) - for i in range(1, len(records)): - table = vstack([table, self.orbit_from_record(records[i])[1]]) - return table + orbits = [self.orbit_from_record(rec) for rec in records] + tbl = Table(orbits) + return tbl def orbit_from_record(self, record): """Return :py:class:`~astropy.table.Table` given a record. From e07b55fa336c9d976142bb9f241816bd3c3d7bac Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Tue, 5 Feb 2019 22:29:32 +0530 Subject: [PATCH 25/28] Fix PEP8 issues! --- astroquery/dastcom5/core.py | 3 ++- astroquery/dastcom5/tests/test_dastcom5.py | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index c9e886c170..5109eb576b 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -62,7 +62,8 @@ def download_dastcom5(self): print("Downloading datscom5.zip") with self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip") as t: - urllib.request.urlretrieve(self.ftp_url + "dastcom5.zip", filename=dastcom5_zip_path, reporthook=t.update_to) + urllib.request.urlretrieve( + self.ftp_url + "dastcom5.zip", filename=dastcom5_zip_path, reporthook=t.update_to) with zipfile.ZipFile(dastcom5_zip_path) as myzip: myzip.extractall(self.local_path) diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py index 36e163f883..b8045c5046 100644 --- a/astroquery/dastcom5/tests/test_dastcom5.py +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -38,7 +38,8 @@ def test_read_headers(mocker): def test_read_record(mocker): mock_np_fromfile = mocker.patch("poliastro.neos.dastcom5.np.fromfile") mock_open = mocker.patch("poliastro.neos.dastcom5.open") - mock_read_headers = mocker.patch("astroquery.dastcom5.Dastcom5.read_headers") + mock_read_headers = mocker.patch( + "astroquery.dastcom5.Dastcom5.read_headers") mocked_ast_headers = np.array( [(3184, -1, b"00740473", b"00496815")], dtype=[ @@ -88,7 +89,8 @@ def test_download_dastcom5_creates_folder(mocker): def test_download_dastcom5_downloads_file(mocker): - mock_request = mocker.patch("astroquery.dastcom5.urllib.request.urlretrieve") + mock_request = mocker.patch( + "astroquery.dastcom5.urllib.request.urlretrieve") mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") mock_isdir.side_effect = lambda x: x == Dastcom5.local_path @@ -97,5 +99,6 @@ def test_download_dastcom5_downloads_file(mocker): mock_request.assert_called_once_with( Dastcom5.ftp_url + "dastcom5.zip", filename=os.path.join(Dastcom5.local_path, "dastcom5.zip"), - reporthook=Dastcom5.self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip").update_to, + reporthook=Dastcom5.self._show_download_progress( + unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip").update_to, ) From 83c32a0dab233ceb789b9c218ea703766187a83c Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Wed, 6 Feb 2019 22:05:06 +0530 Subject: [PATCH 26/28] Remove urllib, remove class, add _download_file --- astroquery/dastcom5/core.py | 16 ++-------------- astroquery/dastcom5/tests/test_dastcom5.py | 12 +++++------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/astroquery/dastcom5/core.py b/astroquery/dastcom5/core.py index 5109eb576b..21a5231081 100755 --- a/astroquery/dastcom5/core.py +++ b/astroquery/dastcom5/core.py @@ -1,11 +1,9 @@ from __future__ import print_function import re import os -import urllib.request import zipfile import numpy as np import pandas as pd -from tqdm import tqdm from astropy.time import Time from astropy.table import Table @@ -39,7 +37,6 @@ def __init__(self, spk_id=None, name=None): self.com_dtype = conf.COM_DTYPE self.ast_dtype = conf.AST_DTYPE self.ftp_url = conf.FTP_DB_URL - self._show_download_progress = self._Show_Download_Progress def download_dastcom5(self): """Downloads DASTCOM5 database. @@ -50,6 +47,7 @@ def download_dastcom5(self): dastcom5_dir = os.path.join(self.local_path, "dastcom5") dastcom5_zip_path = os.path.join(self.local_path, "dastcom5.zip") + ftp_path = self.ftp_url + "dastcom5.zip" if os.path.isdir(dastcom5_dir): raise FileExistsError( @@ -61,21 +59,11 @@ def download_dastcom5(self): os.makedirs(self.local_path) print("Downloading datscom5.zip") - with self._show_download_progress(unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip") as t: - urllib.request.urlretrieve( - self.ftp_url + "dastcom5.zip", filename=dastcom5_zip_path, reporthook=t.update_to) + self._download_file(url=ftp_path, local_filepath=dastcom5_zip_path) with zipfile.ZipFile(dastcom5_zip_path) as myzip: myzip.extractall(self.local_path) - class _Show_Download_Progress(tqdm): - """Helper class for displaying download progress bar when using download_dastcom5() function - """ - def update_to(self, b=1, bsize=1, tsize=None): - if tsize is not None: - self.total = tsize - self.update(b * bsize - self.n) - def asteroid_db(self): """Return complete DASTCOM5 asteroid database. diff --git a/astroquery/dastcom5/tests/test_dastcom5.py b/astroquery/dastcom5/tests/test_dastcom5.py index b8045c5046..37b67d8dc8 100644 --- a/astroquery/dastcom5/tests/test_dastcom5.py +++ b/astroquery/dastcom5/tests/test_dastcom5.py @@ -89,16 +89,14 @@ def test_download_dastcom5_creates_folder(mocker): def test_download_dastcom5_downloads_file(mocker): - mock_request = mocker.patch( - "astroquery.dastcom5.urllib.request.urlretrieve") + mock_download = mocker.patch( + "astroquery.query.BaseQuery._download_file") mock_isdir = mocker.patch("astroquery.dastcom5.os.path.isdir") mock_zipfile = mocker.patch("astroquery.dastcom5.zipfile") mock_isdir.side_effect = lambda x: x == Dastcom5.local_path mock_zipfile.is_zipfile.return_value = False Dastcom5.download_dastcom5() - mock_request.assert_called_once_with( - Dastcom5.ftp_url + "dastcom5.zip", - filename=os.path.join(Dastcom5.local_path, "dastcom5.zip"), - reporthook=Dastcom5.self._show_download_progress( - unit='B', unit_scale=True, miniters=1, desc="dastcom5.zip").update_to, + mock_download.assert_called_once_with( + url=Dastcom5.ftp_url + "dastcom5.zip", + local_filepath=os.path.join(Dastcom5.local_path, "dastcom5.zip"), ) From 9be0535084db72b07d1008207a6364b0787c1012 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 7 Feb 2019 23:03:49 +0530 Subject: [PATCH 27/28] Add ftp_download method --- astroquery/query.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/astroquery/query.py b/astroquery/query.py index 52f947b9b1..9227c64704 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -10,6 +10,7 @@ import io import os import requests +import urllib.request import six from astropy.config import paths @@ -248,6 +249,8 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None, Download a file. Resembles `astropy.utils.data.download_file` but uses the local ``_session`` """ + if url[:3] == 'ftp': + return self._ftp_download(url, local_filepath) if head_safe: response = self._session.request("HEAD", url, timeout=timeout, stream=True, @@ -342,6 +345,9 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None, response.close() return response + def __ftp_download(self, url, local_filepath): + urllib.request.urlretrieve(url, local_filepath) + class suspend_cache: """ From 10d497fd76469fbba45fad5821b91e38b90c8c73 Mon Sep 17 00:00:00 2001 From: Shreyas Bapat Date: Thu, 7 Feb 2019 23:23:22 +0530 Subject: [PATCH 28/28] Remove _ from download ftp --- astroquery/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/query.py b/astroquery/query.py index 9227c64704..ade5f51155 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -345,7 +345,7 @@ def _download_file(self, url, local_filepath, timeout=None, auth=None, response.close() return response - def __ftp_download(self, url, local_filepath): + def _ftp_download(self, url, local_filepath): urllib.request.urlretrieve(url, local_filepath)