"""
Functions for reading and retrieving data from NSRDB PSM4. See:
https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-GOES-aggregated-v4-0-0-download/
https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-GOES-tmy-v4-0-0-download/
https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-GOES-conus-v4-0-0-download/
https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-GOES-full-disc-v4-0-0-download/
https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-polar-v4-0-0-download/
https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-polar-tmy-v4-0-0-download/
"""
import csv
import io
from urllib.parse import urljoin
import requests
import pandas as pd
from json import JSONDecodeError
from pvlib import tools
NSRDB_API_BASE = "https://developer.nlr.gov/api/nsrdb/v2/solar/"
PSM4_AGG_ENDPOINT = "nsrdb-GOES-aggregated-v4-0-0-download.csv"
PSM4_TMY_ENDPOINT = "nsrdb-GOES-tmy-v4-0-0-download.csv"
PSM4_CON_ENDPOINT = "nsrdb-GOES-conus-v4-0-0-download.csv"
PSM4_FUL_ENDPOINT = "nsrdb-GOES-full-disc-v4-0-0-download.csv"
PSM4_POLAR_ENDPOINT = "nsrdb-polar-v4-0-0-download.csv"
PSM4_POLAR_TMY_ENDPOINT = "nsrdb-polar-tmy-v4-0-0-download.csv"
PSM4_AGG_URL = urljoin(NSRDB_API_BASE, PSM4_AGG_ENDPOINT)
PSM4_TMY_URL = urljoin(NSRDB_API_BASE, PSM4_TMY_ENDPOINT)
PSM4_CON_URL = urljoin(NSRDB_API_BASE, PSM4_CON_ENDPOINT)
PSM4_FUL_URL = urljoin(NSRDB_API_BASE, PSM4_FUL_ENDPOINT)
PSM4_POLAR_URL = urljoin(NSRDB_API_BASE, PSM4_POLAR_ENDPOINT)
PSM4_POLAR_TMY_URL = urljoin(NSRDB_API_BASE, PSM4_POLAR_TMY_ENDPOINT)
PARAMETERS = (
'air_temperature', 'dew_point', 'dhi', 'dni', 'ghi', 'surface_albedo',
'surface_pressure', 'wind_direction', 'wind_speed')
PVLIB_PYTHON = 'pvlib python'
# Dictionary mapping PSM4 response names to pvlib names
VARIABLE_MAP = {
'GHI': 'ghi',
'DHI': 'dhi',
'DNI': 'dni',
'Clearsky GHI': 'ghi_clear',
'Clearsky DHI': 'dhi_clear',
'Clearsky DNI': 'dni_clear',
'Solar Zenith Angle': 'solar_zenith',
'Temperature': 'temp_air',
'Dew Point': 'temp_dew',
'Relative Humidity': 'relative_humidity',
'Pressure': 'pressure',
'Wind Speed': 'wind_speed',
'Wind Direction': 'wind_direction',
'Surface Albedo': 'albedo',
'Precipitable Water': 'precipitable_water',
'AOD': 'aod',
}
# Dictionary mapping pvlib names to PSM4 request names
# Note, PSM4 uses different names for the same variables in the
# response and the request
REQUEST_VARIABLE_MAP = {
'ghi': 'ghi',
'dhi': 'dhi',
'dni': 'dni',
'ghi_clear': 'clearsky_ghi',
'dhi_clear': 'clearsky_dhi',
'dni_clear': 'clearsky_dni',
'solar_zenith': 'solar_zenith_angle',
'temp_air': 'air_temperature',
'temp_dew': 'dew_point',
'relative_humidity': 'relative_humidity',
'pressure': 'surface_pressure',
'wind_speed': 'wind_speed',
'wind_direction': 'wind_direction',
'albedo': 'surface_albedo',
'precipitable_water': 'total_precipitable_water',
'aod': 'aod',
}
[docs]
def get_nsrdb_psm4_aggregated(latitude, longitude, api_key, email,
year, time_step=60,
parameters=PARAMETERS, leap_day=True,
full_name=PVLIB_PYTHON,
affiliation=PVLIB_PYTHON,
utc=False, map_variables=True, url=None,
timeout=30):
"""
Retrieve NSRDB PSM4 timeseries weather data from the PSM4 NSRDB GOES
Aggregated v4 API.
The NSRDB is described in [1]_ and the PSM4 NSRDB GOES Aggregated v4 API is
described in [2]_.
Parameters
----------
latitude : float or int
in decimal degrees, between -90 and 90, north is positive
longitude : float or int
in decimal degrees, between -180 and 180, east is positive
api_key : str
NLR Developer Network API key
email : str
NLR API uses this to automatically communicate messages back
to the user only if necessary
year : int or str
PSM4 API parameter specifing year (e.g. ``2023``) to download. The
allowed values update periodically, so consult the NSRDB reference
below for the current set of options. Called ``names`` in NSRDB API.
time_step : int, {60, 30}
time step in minutes, must be 60 or 30 for PSM4 Aggregated. Called
``interval`` in NSRDB API.
parameters : list of str, optional
meteorological fields to fetch. If not specified, defaults to
``pvlib.iotools.psm4.PARAMETERS``. See reference [2]_ for a list of
available fields. Alternatively, pvlib names may also be used (e.g.
'ghi' rather than 'GHI'); see :const:`REQUEST_VARIABLE_MAP`. To
retrieve all available fields, set ``parameters=[]``.
leap_day : bool, default : True
include leap day in the results
full_name : str, default 'pvlib python'
optional
affiliation : str, default 'pvlib python'
optional
utc: bool, default : False
retrieve data with timestamps converted to UTC. False returns
timestamps in local standard time of the selected location
map_variables : bool, default True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
url : str, optional
Full API endpoint URL. If not specified, the PSM4 GOES Aggregated v4
URL is used.
timeout : int, default 30
time in seconds to wait for server response before timeout
Returns
-------
data : pandas.DataFrame
timeseries data from NLR PSM4
metadata : dict
metadata from NLR PSM4 about the record, see
:func:`pvlib.iotools.read_nsrdb_psm4` for fields
Raises
------
requests.HTTPError
if the request response status is not ok, then the ``'errors'`` field
from the JSON response or any error message in the content will be
raised as an exception, for example if the `api_key` was rejected or if
the coordinates were not found in the NSRDB
Notes
-----
The required NLR developer key, `api_key`, is available for free by
registering at the `NLR Developer Network <https://developer.nlr.gov/>`_.
.. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may
result in rejected requests.
.. warning:: PSM4 is limited to data found in the NSRDB, please consult
the references below for locations with available data.
See Also
--------
pvlib.iotools.get_nsrdb_psm4_tmy, pvlib.iotools.get_nsrdb_psm4_conus,
pvlib.iotools.get_nsrdb_psm4_full_disc, pvlib.iotools.get_nsrdb_psm4_polar,
pvlib.iotools.get_nsrdb_psm4_polar_tmy, pvlib.iotools.read_nsrdb_psm4
References
----------
.. [1] `NLR National Solar Radiation Database (NSRDB)
<https://developer.nlr.gov/docs/solar/nsrdb/>`_
.. [2] `NSRDB GOES Aggregated V4.0.0
<https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-GOES-aggregated-v4-0-0-download/>`_
"""
# The well know text (WKT) representation of geometry notation is strict.
# A POINT object is a string with longitude first, then the latitude, with
# four decimals each, and exactly one space between them.
longitude = ('%9.4f' % longitude).strip()
latitude = ('%8.4f' % latitude).strip()
# TODO: make format_WKT(object_type, *args) in tools.py
# convert pvlib names in parameters to PSM4 convention
parameters = [REQUEST_VARIABLE_MAP.get(a, a) for a in parameters]
# required query-string parameters for request to PSM4 API
params = {
'api_key': api_key,
'full_name': full_name,
'email': email,
'affiliation': affiliation,
'reason': PVLIB_PYTHON,
'mailing_list': 'false',
'wkt': 'POINT(%s %s)' % (longitude, latitude),
'names': year,
'attributes': ','.join(parameters),
'leap_day': str(leap_day).lower(),
'utc': str(utc).lower(),
'interval': time_step
}
# request CSV download from NLR PSM4
if url is None:
url = PSM4_AGG_URL
response = requests.get(url, params=params, timeout=timeout)
if not response.ok:
# if the API key is rejected, then the response status will be 403
# Forbidden, and then the error is in the content and there is no JSON
try:
errors = response.json()['errors']
except JSONDecodeError:
errors = response.content.decode('utf-8')
raise requests.HTTPError(errors, response=response)
# the CSV is in the response content as a UTF-8 bytestring
# to use pandas we need to create a file buffer from the response
fbuf = io.StringIO(response.content.decode('utf-8'))
return read_nsrdb_psm4(fbuf, map_variables)
[docs]
def get_nsrdb_psm4_tmy(latitude, longitude, api_key, email, year='tmy',
time_step=60, parameters=PARAMETERS, leap_day=False,
full_name=PVLIB_PYTHON, affiliation=PVLIB_PYTHON,
utc=False, map_variables=True, url=None, timeout=30):
"""
Retrieve NSRDB PSM4 timeseries weather data from the PSM4 NSRDB GOES
TMY v4 API.
The NSRDB is described in [1]_ and the PSM4 NSRDB GOES TMY v4 API is
described in [2]_.
Parameters
----------
latitude : float or int
in decimal degrees, between -90 and 90, north is positive
longitude : float or int
in decimal degrees, between -180 and 180, east is positive
api_key : str
NLR Developer Network API key
email : str
NLR API uses this to automatically communicate messages back
to the user only if necessary
year : str, default 'tmy'
PSM4 API parameter specifing TMY variant to download (e.g. ``'tmy'``
or ``'tgy-2022'``). The allowed values update periodically, so
consult the NSRDB references below for the current set of options.
Called ``names`` in NSRDB API.
time_step : int, {60}
time step in minutes. Must be 60 for typical year requests. Called
``interval`` in NSRDB API.
parameters : list of str, optional
meteorological fields to fetch. If not specified, defaults to
``pvlib.iotools.psm4.PARAMETERS``. See reference [2]_ for a list of
available fields. Alternatively, pvlib names may also be used (e.g.
'ghi' rather than 'GHI'); see :const:`REQUEST_VARIABLE_MAP`. To
retrieve all available fields, set ``parameters=[]``.
leap_day : bool, default : False
Include leap day in the results. Ignored for tmy/tgy/tdy requests.
full_name : str, default 'pvlib python'
optional
affiliation : str, default 'pvlib python'
optional
utc: bool, default : False
retrieve data with timestamps converted to UTC. False returns
timestamps in local standard time of the selected location
map_variables : bool, default True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
url : str, optional
Full API endpoint URL. If not specified, the PSM4 GOES TMY v4 URL is
used.
timeout : int, default 30
time in seconds to wait for server response before timeout
Returns
-------
data : pandas.DataFrame
timeseries data from NLR PSM4
metadata : dict
metadata from NLR PSM4 about the record, see
:func:`pvlib.iotools.read_nsrdb_psm4` for fields
Raises
------
requests.HTTPError
if the request response status is not ok, then the ``'errors'`` field
from the JSON response or any error message in the content will be
raised as an exception, for example if the `api_key` was rejected or if
the coordinates were not found in the NSRDB
Notes
-----
The required NLR developer key, `api_key`, is available for free by
registering at the `NLR Developer Network <https://developer.nlr.gov/>`_.
.. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may
result in rejected requests.
.. warning:: PSM4 is limited to data found in the NSRDB, please consult
the references below for locations with available data.
See Also
--------
pvlib.iotools.get_nsrdb_psm4_polar_tmy,
pvlib.iotools.get_nsrdb_psm4_aggregated,
pvlib.iotools.get_nsrdb_psm4_conus, pvlib.iotools.get_nsrdb_psm4_full_disc,
pvlib.iotools.get_nsrdb_psm4_polar,
pvlib.iotools.read_nsrdb_psm4
References
----------
.. [1] `NLR National Solar Radiation Database (NSRDB)
<https://developer.nlr.gov/docs/solar/nsrdb/>`_
.. [2] `NSRDB GOES Tmy V4.0.0
<https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-GOES-tmy-v4-0-0-download/>`_
"""
# The well know text (WKT) representation of geometry notation is strict.
# A POINT object is a string with longitude first, then the latitude, with
# four decimals each, and exactly one space between them.
longitude = ('%9.4f' % longitude).strip()
latitude = ('%8.4f' % latitude).strip()
# TODO: make format_WKT(object_type, *args) in tools.py
# convert pvlib names in parameters to PSM4 convention
parameters = [REQUEST_VARIABLE_MAP.get(a, a) for a in parameters]
# required query-string parameters for request to PSM4 API
params = {
'api_key': api_key,
'full_name': full_name,
'email': email,
'affiliation': affiliation,
'reason': PVLIB_PYTHON,
'mailing_list': 'false',
'wkt': 'POINT(%s %s)' % (longitude, latitude),
'names': year,
'attributes': ','.join(parameters),
'leap_day': str(leap_day).lower(),
'utc': str(utc).lower(),
'interval': time_step
}
# request CSV download from NLR PSM4
if url is None:
url = PSM4_TMY_URL
response = requests.get(url, params=params, timeout=timeout)
if not response.ok:
# if the API key is rejected, then the response status will be 403
# Forbidden, and then the error is in the content and there is no JSON
try:
errors = response.json()['errors']
except JSONDecodeError:
errors = response.content.decode('utf-8')
raise requests.HTTPError(errors, response=response)
# the CSV is in the response content as a UTF-8 bytestring
# to use pandas we need to create a file buffer from the response
fbuf = io.StringIO(response.content.decode('utf-8'))
return read_nsrdb_psm4(fbuf, map_variables)
[docs]
def get_nsrdb_psm4_conus(latitude, longitude, api_key, email, year,
time_step=60, parameters=PARAMETERS, leap_day=True,
full_name=PVLIB_PYTHON, affiliation=PVLIB_PYTHON,
utc=False, map_variables=True, url=None, timeout=30):
"""
Retrieve NSRDB PSM4 timeseries weather data from the PSM4 NSRDB GOES CONUS
v4 API.
The NSRDB is described in [1]_ and the PSM4 NSRDB GOES CONUS v4 API is
described in [2]_.
Parameters
----------
latitude : float or int
in decimal degrees, between -90 and 90, north is positive
longitude : float or int
in decimal degrees, between -180 and 180, east is positive
api_key : str
NLR Developer Network API key
email : str
NLR API uses this to automatically communicate messages back
to the user only if necessary
year : int or str
PSM4 API parameter specifing year (e.g. ``2023``) to download. The
allowed values update periodically, so consult the NSRDB reference
below for the current set of options. Called ``names`` in NSRDB API.
time_step : int, {60, 5, 15, 30}
time step in minutes. Called ``interval`` in NSRDB API.
parameters : list of str, optional
meteorological fields to fetch. If not specified, defaults to
``pvlib.iotools.psm4.PARAMETERS``. See reference [2]_ for a list of
available fields. Alternatively, pvlib names may also be used (e.g.
'ghi' rather than 'GHI'); see :const:`REQUEST_VARIABLE_MAP`. To
retrieve all available fields, set ``parameters=[]``.
leap_day : bool, default : True
include leap day in the results
full_name : str, default 'pvlib python'
optional
affiliation : str, default 'pvlib python'
optional
utc: bool, default : False
retrieve data with timestamps converted to UTC. False returns
timestamps in local standard time of the selected location
map_variables : bool, default True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
url : str, optional
Full API endpoint URL. If not specified, the PSM4 GOES CONUS v4 URL is
used.
timeout : int, default 30
time in seconds to wait for server response before timeout
Returns
-------
data : pandas.DataFrame
timeseries data from NLR PSM4
metadata : dict
metadata from NLR PSM4 about the record, see
:func:`pvlib.iotools.read_nsrdb_psm4` for fields
Raises
------
requests.HTTPError
if the request response status is not ok, then the ``'errors'`` field
from the JSON response or any error message in the content will be
raised as an exception, for example if the `api_key` was rejected or if
the coordinates were not found in the NSRDB
Notes
-----
The required NLR developer key, `api_key`, is available for free by
registering at the `NLR Developer Network <https://developer.nlr.gov/>`_.
.. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may
result in rejected requests.
.. warning:: PSM4 is limited to data found in the NSRDB, please consult
the references below for locations with available data.
See Also
--------
pvlib.iotools.get_nsrdb_psm4_aggregated, pvlib.iotools.get_nsrdb_psm4_tmy,
pvlib.iotools.get_nsrdb_psm4_full_disc, pvlib.iotools.get_nsrdb_psm4_polar,
pvlib.iotools.get_nsrdb_psm4_polar_tmy, pvlib.iotools.read_nsrdb_psm4
References
----------
.. [1] `NLR National Solar Radiation Database (NSRDB)
<https://developer.nlr.gov/docs/solar/nsrdb/>`_
.. [2] `NSRDB GOES Conus V4.0.0
<https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-GOES-conus-v4-0-0-download/>`_
"""
# The well know text (WKT) representation of geometry notation is strict.
# A POINT object is a string with longitude first, then the latitude, with
# four decimals each, and exactly one space between them.
longitude = ('%9.4f' % longitude).strip()
latitude = ('%8.4f' % latitude).strip()
# TODO: make format_WKT(object_type, *args) in tools.py
# convert pvlib names in parameters to PSM4 convention
parameters = [REQUEST_VARIABLE_MAP.get(a, a) for a in parameters]
# required query-string parameters for request to PSM4 API
params = {
'api_key': api_key,
'full_name': full_name,
'email': email,
'affiliation': affiliation,
'reason': PVLIB_PYTHON,
'mailing_list': 'false',
'wkt': 'POINT(%s %s)' % (longitude, latitude),
'names': year,
'attributes': ','.join(parameters),
'leap_day': str(leap_day).lower(),
'utc': str(utc).lower(),
'interval': time_step
}
# request CSV download from NLR PSM4
if url is None:
url = PSM4_CON_URL
response = requests.get(url, params=params, timeout=timeout)
if not response.ok:
# if the API key is rejected, then the response status will be 403
# Forbidden, and then the error is in the content and there is no JSON
try:
errors = response.json()['errors']
except JSONDecodeError:
errors = response.content.decode('utf-8')
raise requests.HTTPError(errors, response=response)
# the CSV is in the response content as a UTF-8 bytestring
# to use pandas we need to create a file buffer from the response
fbuf = io.StringIO(response.content.decode('utf-8'))
return read_nsrdb_psm4(fbuf, map_variables)
[docs]
def get_nsrdb_psm4_full_disc(latitude, longitude, api_key, email,
year, time_step=60,
parameters=PARAMETERS, leap_day=True,
full_name=PVLIB_PYTHON,
affiliation=PVLIB_PYTHON, utc=False,
map_variables=True, url=None, timeout=30):
"""
Retrieve NSRDB PSM4 timeseries weather data from the PSM4 NSRDB GOES Full
Disc v4 API.
The NSRDB is described in [1]_ and the PSM4 NSRDB GOES Full Disc v4 API is
described in [2]_.
Parameters
----------
latitude : float or int
in decimal degrees, between -90 and 90, north is positive
longitude : float or int
in decimal degrees, between -180 and 180, east is positive
api_key : str
NLR Developer Network API key
email : str
NLR API uses this to automatically communicate messages back
to the user only if necessary
year : int or str
PSM4 API parameter specifing year (e.g. ``2023``) to download. The
allowed values update periodically, so consult the NSRDB reference
below for the current set of options. Called ``names`` in NSRDB API.
time_step : int, {60, 10, 30}
time step in minutes, must be 10, 30 or 60. Called ``interval`` in
NSRDB API.
parameters : list of str, optional
meteorological fields to fetch. If not specified, defaults to
``pvlib.iotools.psm4.PARAMETERS``. See reference [2]_ for a list of
available fields. Alternatively, pvlib names may also be used (e.g.
'ghi' rather than 'GHI'); see :const:`REQUEST_VARIABLE_MAP`. To
retrieve all available fields, set ``parameters=[]``.
leap_day : bool, default : True
include leap day in the results
full_name : str, default 'pvlib python'
optional
affiliation : str, default 'pvlib python'
optional
utc: bool, default : False
retrieve data with timestamps converted to UTC. False returns
timestamps in local standard time of the selected location
map_variables : bool, default True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
url : str, optional
Full API endpoint URL. If not specified, the PSM4 GOES Full Disc v4
URL is used.
timeout : int, default 30
time in seconds to wait for server response before timeout
Returns
-------
data : pandas.DataFrame
timeseries data from NLR PSM4
metadata : dict
metadata from NLR PSM4 about the record, see
:func:`pvlib.iotools.read_nsrdb_psm4` for fields
Raises
------
requests.HTTPError
if the request response status is not ok, then the ``'errors'`` field
from the JSON response or any error message in the content will be
raised as an exception, for example if the `api_key` was rejected or if
the coordinates were not found in the NSRDB
Notes
-----
The required NLR developer key, `api_key`, is available for free by
registering at the `NLR Developer Network <https://developer.nlr.gov/>`_.
.. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may
result in rejected requests.
.. warning:: PSM4 is limited to data found in the NSRDB, please consult
the references below for locations with available data.
See Also
--------
pvlib.iotools.get_nsrdb_psm4_aggregated, pvlib.iotools.get_nsrdb_psm4_tmy,
pvlib.iotools.get_nsrdb_psm4_conus, pvlib.iotools.get_nsrdb_psm4_polar,
pvlib.iotools.get_nsrdb_psm4_polar_tmy, pvlib.iotools.read_nsrdb_psm4
References
----------
.. [1] `NLR National Solar Radiation Database (NSRDB)
<https://developer.nlr.gov/docs/solar/nsrdb/>`_
.. [2] `NSRDB GOES Full Disc V4.0.0
<https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-GOES-full-disc-v4-0-0-download/>`_
"""
# The well know text (WKT) representation of geometry notation is strict.
# A POINT object is a string with longitude first, then the latitude, with
# four decimals each, and exactly one space between them.
longitude = ('%9.4f' % longitude).strip()
latitude = ('%8.4f' % latitude).strip()
# TODO: make format_WKT(object_type, *args) in tools.py
# convert pvlib names in parameters to PSM4 convention
parameters = [REQUEST_VARIABLE_MAP.get(a, a) for a in parameters]
# required query-string parameters for request to PSM4 API
params = {
'api_key': api_key,
'full_name': full_name,
'email': email,
'affiliation': affiliation,
'reason': PVLIB_PYTHON,
'mailing_list': 'false',
'wkt': 'POINT(%s %s)' % (longitude, latitude),
'names': year,
'attributes': ','.join(parameters),
'leap_day': str(leap_day).lower(),
'utc': str(utc).lower(),
'interval': time_step
}
# request CSV download from NLR PSM4
if url is None:
url = PSM4_FUL_URL
response = requests.get(url, params=params, timeout=timeout)
if not response.ok:
# if the API key is rejected, then the response status will be 403
# Forbidden, and then the error is in the content and there is no JSON
try:
errors = response.json()['errors']
except JSONDecodeError:
errors = response.content.decode('utf-8')
raise requests.HTTPError(errors, response=response)
# the CSV is in the response content as a UTF-8 bytestring
# to use pandas we need to create a file buffer from the response
fbuf = io.StringIO(response.content.decode('utf-8'))
return read_nsrdb_psm4(fbuf, map_variables)
[docs]
def get_nsrdb_psm4_polar(latitude, longitude, api_key, email,
year, *, time_step=60,
parameters=PARAMETERS, leap_day=True,
full_name=PVLIB_PYTHON,
affiliation=PVLIB_PYTHON, utc=False,
map_variables=True, url=None, timeout=30):
"""
Retrieve timeseries weather data from the PSM4 NSRDB Polar API.
The NSRDB is described in [1]_ and the NSRDB PSM4 NSRDB Polar v4 dataset
is described in [2]_, [3]_. The polar dataset extends NSRDB coverage to
high latitude regions, that are outside the field of view of the
geostationary satellites.
The dataset provides data from 2013 onwards for the exclusive economic
zones above 60 degrees north. The data have a spatial resolution of 4 km
and a temporal resolution of 1 hour.
Parameters
----------
latitude : float or int
in decimal degrees, between 60 and 90 degrees north.
Location must be within an exclusive economic zone.
longitude : float or int
in decimal degrees, between -180 and 180, east is positive.
Location must be within an exclusive economic zone.
api_key : str
NLR Developer Network API key
email : str
NLR API uses this to automatically communicate messages back
to the user only if necessary
year : int or str
PSM4 API parameter specifing year (e.g. ``2023``) to download. The
allowed values update periodically, so consult the NSRDB reference
below for the current set of options. Called ``names`` in NSRDB API.
time_step : int, default 60
time step in minutes. Called ``interval`` in NSRDB API.
Only 60 is supported for the Polar dataset.
parameters : list of str, optional
meteorological fields to fetch. If not specified, defaults to
``pvlib.iotools.psm4.PARAMETERS``. See reference [2]_ for a list of
available fields. Alternatively, pvlib names may also be used (e.g.
'ghi' rather than 'GHI'); see :const:`REQUEST_VARIABLE_MAP`. To
retrieve all available fields, set ``parameters=[]``.
leap_day : bool, default : True
include leap day in the results
full_name : str, default 'pvlib python'
optional
affiliation : str, default 'pvlib python'
optional
utc: bool, default : False
retrieve data with timestamps converted to UTC. False returns
timestamps in local standard time of the selected location
map_variables : bool, default True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
url : str, optional
Full API endpoint URL. If not specified, the PSM4 Polar v4 URL is
used.
timeout : int, default 30
time in seconds to wait for server response before timeout
Returns
-------
data : pandas.DataFrame
timeseries data from NLR PSM4 Polar
metadata : dict
metadata from NLR PSM4 about the record, see
:func:`pvlib.iotools.read_nsrdb_psm4` for fields
Raises
------
requests.HTTPError
if the request response status is not ok, then the ``'errors'`` field
from the JSON response or any error message in the content will be
raised as an exception, for example if the `api_key` was rejected or if
the coordinates were not found in the NSRDB
Notes
-----
The required NLR developer key, `api_key`, is available for free by
registering at the `NLR Developer Network <https://developer.nlr.gov/>`_.
.. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may
result in rejected requests.
.. warning:: PSM4 is limited to data found in the NSRDB, please consult
the references below for locations with available data.
See Also
--------
pvlib.iotools.get_nsrdb_psm4_aggregated, pvlib.iotools.get_nsrdb_psm4_tmy,
pvlib.iotools.get_nsrdb_psm4_conus, pvlib.iotools.get_nsrdb_psm4_full_disc,
pvlib.iotools.get_nsrdb_psm4_polar, pvlib.iotools.get_nsrdb_psm4_polar_tmy,
pvlib.iotools.read_nsrdb_psm4
References
----------
.. [1] `NLR National Solar Radiation Database (NSRDB)
<https://developer.nlr.gov/docs/solar/nsrdb/>`_
.. [2] `NLR NSRDB Polar Data
<https://nsrdb.nlr.gov/data-sets/polar-data/>`_
.. [3] `NSRDB Polar V4.0.0
<https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-polar-v4-0-0-download/>`_
"""
# The well know text (WKT) representation of geometry notation is strict.
# A POINT object is a string with longitude first, then the latitude, with
# four decimals each, and exactly one space between them.
longitude = ('%9.4f' % longitude).strip()
latitude = ('%8.4f' % latitude).strip()
# TODO: make format_WKT(object_type, *args) in tools.py
# convert pvlib names in parameters to PSM4 convention
parameters = [REQUEST_VARIABLE_MAP.get(a, a) for a in parameters]
# required query-string parameters for request to PSM4 API
params = {
'api_key': api_key,
'full_name': full_name,
'email': email,
'affiliation': affiliation,
'reason': PVLIB_PYTHON,
'mailing_list': 'false',
'wkt': 'POINT(%s %s)' % (longitude, latitude),
'names': year,
'attributes': ','.join(parameters),
'leap_day': str(leap_day).lower(),
'utc': str(utc).lower(),
'interval': time_step
}
# request CSV download from NLR PSM4
if url is None:
url = PSM4_POLAR_URL
response = requests.get(url, params=params, timeout=timeout)
if not response.ok:
# if the API key is rejected, then the response status will be 403
# Forbidden, and then the error is in the content and there is no JSON
try:
errors = response.json()['errors']
except JSONDecodeError:
errors = response.content.decode('utf-8')
raise requests.HTTPError(errors, response=response)
# the CSV is in the response content as a UTF-8 bytestring
# to use pandas we need to create a file buffer from the response
fbuf = io.StringIO(response.content.decode('utf-8'))
return read_nsrdb_psm4(fbuf, map_variables)
[docs]
def get_nsrdb_psm4_polar_tmy(latitude, longitude, api_key, email, year='tmy',
time_step=60, parameters=PARAMETERS,
leap_day=False, full_name=PVLIB_PYTHON,
affiliation=PVLIB_PYTHON, utc=False,
map_variables=True, url=None, timeout=30):
"""
Retrieve timeseries weather data from the PSM4 NSRDB Polar TMY v4 API.
The NSRDB is described in [1]_ and the NSRDB PSM4 Polar TMY v4 dataset is
described in [2]_, [3]_. The polar dataset extends NSRDB coverage to high
latitude polar regions, that are outside the field of view of the
geostationary satellites.
The dataset provides typical meteorological year (TMY) data for the
exclusive economic zones above 60 degrees north. The data have a spatial
resolution of 4 km and a temporal resolution of 1 hour.
Parameters
----------
latitude : float or int
in decimal degrees, between 60 and 90 degrees north.
Location must be within an exclusive economic zone.
longitude : float or int
in decimal degrees, between -180 and 180, east is positive.
Location must be within an exclusive economic zone.
api_key : str
NLR Developer Network API key
email : str
NLR API uses this to automatically communicate messages back
to the user only if necessary
year : str, default 'tmy'
PSM4 API parameter specifing TMY variant to download (e.g. ``'tmy'``
or ``'tgy-2023'``). The allowed values update periodically, so
consult the NSRDB references below for the current set of options.
Called ``names`` in NSRDB API.
time_step : int, {60}
time step in minutes. Must be 60 for typical year requests. Called
``interval`` in NSRDB API.
parameters : list of str, optional
meteorological fields to fetch. If not specified, defaults to
``pvlib.iotools.psm4.PARAMETERS``. See reference [2]_ for a list of
available fields. Alternatively, pvlib names may also be used (e.g.
'ghi' rather than 'GHI'); see :const:`REQUEST_VARIABLE_MAP`. To
retrieve all available fields, set ``parameters=[]``.
leap_day : bool, default : False
Include leap day in the results. Ignored for tmy/tgy/tdy requests.
full_name : str, default 'pvlib python'
optional
affiliation : str, default 'pvlib python'
optional
utc: bool, default : False
retrieve data with timestamps converted to UTC. False returns
timestamps in local standard time of the selected location
map_variables : bool, default True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
url : str, optional
Full API endpoint URL. If not specified, the PSM4 Polar TMY v4 URL
is used.
timeout : int, default 30
time in seconds to wait for server response before timeout
Returns
-------
data : pandas.DataFrame
timeseries data from NLR PSM4 Polar TMY
metadata : dict
metadata from NLR PSM4, see
:func:`pvlib.iotools.read_nsrdb_psm4` for fields
Raises
------
requests.HTTPError
if the request response status is not ok, then the ``'errors'`` field
from the JSON response or any error message in the content will be
raised as an exception, for example if the `api_key` was rejected or if
the coordinates were not found in the NSRDB
Notes
-----
The required NLR developer key, `api_key`, is available for free by
registering at the `NLR Developer Network <https://developer.nlr.gov/>`_.
.. warning:: The "DEMO_KEY" `api_key` is severely rate limited and may
result in rejected requests.
.. warning:: PSM4 is limited to data found in the NSRDB, please consult
the references below for locations with available data.
See Also
--------
pvlib.iotools.get_nsrdb_psm4_polar, pvlib.iotools.get_nsrdb_psm4_tmy,
pvlib.iotools.get_nsrdb_psm4_aggregated,
pvlib.iotools.get_nsrdb_psm4_conus,
pvlib.iotools.get_nsrdb_psm4_full_disc, pvlib.iotools.read_nsrdb_psm4
References
----------
.. [1] `NLR National Solar Radiation Database (NSRDB)
<https://developer.nlr.gov/docs/solar/nsrdb/>`_
.. [2] `NLR NSRDB Polar Data
<https://nsrdb.nlr.gov/data-sets/polar-data/>`_
.. [3] `NSRDB Polar Tmy V4.0.0
<https://developer.nlr.gov/docs/solar/nsrdb/nsrdb-polar-tmy-v4-0-0-download/>`_
"""
# The well know text (WKT) representation of geometry notation is strict.
# A POINT object is a string with longitude first, then the latitude, with
# four decimals each, and exactly one space between them.
longitude = ('%9.4f' % longitude).strip()
latitude = ('%8.4f' % latitude).strip()
# TODO: make format_WKT(object_type, *args) in tools.py
# convert pvlib names in parameters to PSM4 convention
parameters = [REQUEST_VARIABLE_MAP.get(a, a) for a in parameters]
# required query-string parameters for request to PSM4 API
params = {
'api_key': api_key,
'full_name': full_name,
'email': email,
'affiliation': affiliation,
'reason': PVLIB_PYTHON,
'mailing_list': 'false',
'wkt': 'POINT(%s %s)' % (longitude, latitude),
'names': year,
'attributes': ','.join(parameters),
'leap_day': str(leap_day).lower(),
'utc': str(utc).lower(),
'interval': time_step
}
# request CSV download from NLR PSM4
if url is None:
url = PSM4_POLAR_TMY_URL
response = requests.get(url, params=params, timeout=timeout)
if not response.ok:
# if the API key is rejected, then the response status will be 403
# Forbidden, and then the error is in the content and there is no JSON
try:
errors = response.json()['errors']
except JSONDecodeError:
errors = response.content.decode('utf-8')
raise requests.HTTPError(errors, response=response)
# the CSV is in the response content as a UTF-8 bytestring
# to use pandas we need to create a file buffer from the response
fbuf = io.StringIO(response.content.decode('utf-8'))
return read_nsrdb_psm4(fbuf, map_variables)
[docs]
def read_nsrdb_psm4(filename, map_variables=True):
"""
Read an NSRDB PSM4 weather file (formatted as SAM CSV).
The NSRDB is described in [1]_ and the SAM CSV format is described in [2]_.
Parameters
----------
filename: str, path-like, or buffer
Filename or in-memory buffer of a file containing data to read.
map_variables: bool, default True
When true, renames columns of the Dataframe to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
Returns
-------
data : pandas.DataFrame
timeseries data from NLR PSM4
metadata : dict
metadata from NLR PSM4 about the record, see notes for fields
Notes
-----
The return is a tuple with two items. The first item is a dataframe with
the PSM4 timeseries data.
The second item is a dictionary with metadata from NLR PSM4 about the
record containing the following fields:
* Source
* Location ID
* City
* State
* Country
* Latitude
* Longitude
* Time Zone
* Elevation
* Local Time Zone
* Clearsky DHI Units
* Clearsky DNI Units
* Clearsky GHI Units
* Dew Point Units
* DHI Units
* DNI Units
* GHI Units
* Solar Zenith Angle Units
* Temperature Units
* Pressure Units
* Relative Humidity Units
* Precipitable Water Units
* Wind Direction Units
* Wind Speed Units
* Cloud Type -15
* Cloud Type 0
* Cloud Type 1
* Cloud Type 2
* Cloud Type 3
* Cloud Type 4
* Cloud Type 5
* Cloud Type 6
* Cloud Type 7
* Cloud Type 8
* Cloud Type 9
* Cloud Type 10
* Cloud Type 11
* Cloud Type 12
* Fill Flag 0
* Fill Flag 1
* Fill Flag 2
* Fill Flag 3
* Fill Flag 4
* Fill Flag 5
* Surface Albedo Units
* Version
Examples
--------
>>> # Read a local PSM4 file:
>>> df, metadata = iotools.read_nsrdb_psm4("data.csv") # doctest: +SKIP
>>> # Read a file object or an in-memory buffer:
>>> with open(filename, 'r') as f: # doctest: +SKIP
... df, metadata = iotools.read_nsrdb_psm4(f) # doctest: +SKIP
See Also
--------
pvlib.iotools.get_nsrdb_psm4_aggregated
pvlib.iotools.get_nsrdb_psm4_tmy
pvlib.iotools.get_nsrdb_psm4_conus
pvlib.iotools.get_nsrdb_psm4_full_disc
pvlib.iotools.get_nsrdb_psm4_polar
pvlib.iotools.get_nsrdb_psm4_polar_tmy
References
----------
.. [1] `NLR National Solar Radiation Database (NSRDB)
<https://developer.nlr.gov/docs/solar/nsrdb/>`_
.. [2] `Standard Time Series Data File Format
<https://web.archive.org/web/20170207203107/https://sam.nrel.gov/sites/default/files/content/documents/pdf/wfcsv.pdf>`_
"""
with tools._file_context_manager(filename) as fbuf:
# The first 3 header lines are parsed with the csv module rather than a
# naive str.split(',') so that quoted fields containing commas are kept
# intact. Spectral-on-demand files, for instance, have column names
# like '"GaAs (Bauhuis et al., 2009)"' whose embedded commas would
# otherwise be split into spurious columns (see GH #2736).
# The first 2 lines of the response are headers with metadata
metadata_fields = next(csv.reader([fbuf.readline()]))
metadata_values = next(csv.reader([fbuf.readline()]))
# get the column names so we can set the dtypes
columns = next(csv.reader([fbuf.readline()]))
columns[-1] = columns[-1].strip() # strip trailing newline
# Since the header has so many columns, excel saves blank cols in the
# data below the header lines.
columns = [col for col in columns if col != '']
dtypes = dict.fromkeys(columns, float)
dtypes.update({'Year': int, 'Month': int, 'Day': int, 'Hour': int,
'Minute': int, 'Cloud Type': int, 'Fill Flag': int})
data = pd.read_csv(
fbuf, header=None, names=columns, usecols=columns, dtype=dtypes,
delimiter=',', lineterminator='\n') # skip carriage returns \r
metadata_fields[-1] = metadata_fields[-1].strip() # trailing newline
metadata_values[-1] = metadata_values[-1].strip() # trailing newline
metadata = dict(zip(metadata_fields, metadata_values))
# the response is all strings, so set some metadata types to numbers
metadata['Local Time Zone'] = int(metadata['Local Time Zone'])
metadata['Time Zone'] = int(metadata['Time Zone'])
metadata['Latitude'] = float(metadata['Latitude'])
metadata['Longitude'] = float(metadata['Longitude'])
metadata['Elevation'] = int(metadata['Elevation'])
# the response 1st 5 columns are a date vector, convert to datetime
dtidx = pd.to_datetime(data[['Year', 'Month', 'Day', 'Hour', 'Minute']])
# in USA all timezones are integers
tz = 'Etc/GMT%+d' % -metadata['Time Zone']
data.index = pd.DatetimeIndex(dtidx).tz_localize(tz)
if map_variables:
data = data.rename(columns=VARIABLE_MAP)
metadata['latitude'] = metadata.pop('Latitude')
metadata['longitude'] = metadata.pop('Longitude')
metadata['altitude'] = metadata.pop('Elevation')
return data, metadata
