"""Functions for reading TMY2 and TMY3 data files."""
import datetime
import re
import pandas as pd
# Dictionary mapping TMY3 names to pvlib names
VARIABLE_MAP = {
'GHI (W/m^2)': 'ghi',
'ETR (W/m^2)': 'ghi_extra',
'DNI (W/m^2)': 'dni',
'ETRN (W/m^2)': 'dni_extra',
'DHI (W/m^2)': 'dhi',
'Pressure (mbar)': 'pressure',
'Wdir (degrees)': 'wind_direction',
'Wspd (m/s)': 'wind_speed',
'Dry-bulb (C)': 'temp_air',
'Dew-point (C)': 'temp_dew',
'RHum (%)': 'relative_humidity',
'Alb (unitless)': 'albedo',
'Pwat (cm)': 'precipitable_water'
}
[docs]
def read_tmy3(filename, coerce_year=None, map_variables=True, encoding=None):
"""Read a TMY3 file into a pandas dataframe.
Note that values contained in the metadata dictionary are unchanged
from the TMY3 file (i.e. units are retained). In the case of any
discrepancies between this documentation and the TMY3 User's Manual
[1]_, the TMY3 User's Manual takes precedence.
The TMY3 files were updated in Jan. 2015. This function requires the
use of the updated files.
Parameters
----------
filename : str
A relative file path or absolute file path.
coerce_year : int, optional
If supplied, the year of the index will be set to ``coerce_year``, except
for the last index value which will be set to the *next* year so that
the index increases monotonically.
map_variables : bool, default True
When True, renames columns of the DataFrame to pvlib variable names
where applicable. See variable :const:`VARIABLE_MAP`.
encoding : str, optional
Encoding of the file. For files that contain non-UTF8 characters it may
be necessary to specify an alternative encoding, e.g., for
SolarAnywhere TMY3 files the encoding should be 'iso-8859-1'. Users
may also consider using the 'utf-8-sig' encoding.
Returns
-------
Tuple of the form (data, metadata).
data : DataFrame
A pandas dataframe with the columns described in the table
below. For more detailed descriptions of each component, please
consult the TMY3 User's Manual [1]_, especially tables 1-1
through 1-6.
metadata : dict
The site metadata available in the file.
Notes
-----
The returned structures have the following fields.
=============== ====== ===================
key format description
=============== ====== ===================
altitude Float site elevation
latitude Float site latitudeitude
longitude Float site longitudeitude
Name String site name
State String state
TZ Float UTC offset
USAF Int USAF identifier
=============== ====== ===================
======================== ======================================================================================================================================================
field description
======================== ======================================================================================================================================================
**† denotes variables that are mapped when `map_variables` is True**
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Index A pandas datetime index. NOTE, the index is timezone aware, and times are set to local standard time (daylight savings is not included)
ghi_extra† Extraterrestrial horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2
dni_extra† Extraterrestrial normal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2
ghi† Direct and diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2
GHI source See [1]_, Table 1-4
GHI uncert (%) Uncertainty based on random and bias error estimates see [2]_
dni† Amount of direct normal radiation (modeled) recv'd during 60 mintues prior to timestamp, Wh/m^2
DNI source See [1]_, Table 1-4
DNI uncert (%) Uncertainty based on random and bias error estimates see [2]_
dhi† Amount of diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2
DHI source See [1]_, Table 1-4
DHI uncert (%) Uncertainty based on random and bias error estimates see [2]_
GH illum (lx) Avg. total horizontal illuminance recv'd during the 60 minutes prior to timestamp, lx
GH illum source See [1]_, Table 1-4
GH illum uncert (%) Uncertainty based on random and bias error estimates see [2]_
DN illum (lx) Avg. direct normal illuminance recv'd during the 60 minutes prior to timestamp, lx
DN illum source See [1]_, Table 1-4
DN illum uncert (%) Uncertainty based on random and bias error estimates see [2]_
DH illum (lx) Avg. horizontal diffuse illuminance recv'd during the 60 minutes prior to timestamp, lx
DH illum source See [1]_, Table 1-4
DH illum uncert (%) Uncertainty based on random and bias error estimates see [2]_
Zenith lum (cd/m^2) Avg. luminance at the sky's zenith during the 60 minutes prior to timestamp, cd/m^2
Zenith lum source See [1]_, Table 1-4
Zenith lum uncert (%) Uncertainty based on random and bias error estimates see [1]_ section 2.10
TotCld (tenths) Amount of sky dome covered by clouds or obscuring phenonema at time stamp, tenths of sky
TotCld source See [1]_, Table 1-5
TotCld uncert (code) See [1]_, Table 1-6
OpqCld (tenths) Amount of sky dome covered by clouds or obscuring phenonema that prevent observing the sky at time stamp, tenths of sky
OpqCld source See [1]_, Table 1-5
OpqCld uncert (code) See [1]_, Table 1-6
temp_air† Dry bulb temperature at the time indicated, deg C
Dry-bulb source See [1]_, Table 1-5
Dry-bulb uncert (code) See [1]_, Table 1-6
temp_dew† Dew-point temperature at the time indicated, deg C
Dew-point source See [1]_, Table 1-5
Dew-point uncert (code) See [1]_, Table 1-6
relative_humidity† Relatitudeive humidity at the time indicated, percent
RHum source See [1]_, Table 1-5
RHum uncert (code) See [1]_, Table 1-6
pressure† Station pressure at the time indicated, 1 mbar
Pressure source See [1]_, Table 1-5
Pressure uncert (code) See [1]_, Table 1-6
wind_direction† Wind direction at time indicated, degrees from north (360 = north; 0 = undefined,calm)
Wdir source See [1]_, Table 1-5
Wdir uncert (code) See [1]_, Table 1-6
wind_speed† Wind speed at the time indicated, meter/second
Wspd source See [1]_, Table 1-5
Wspd uncert (code) See [1]_, Table 1-6
Hvis (m) Distance to discernable remote objects at time indicated (7777=unlimited), meter
Hvis source See [1]_, Table 1-5
Hvis uncert (coe) See [1]_, Table 1-6
CeilHgt (m) Height of cloud base above local terrain (7777=unlimited), meter
CeilHgt source See [1]_, Table 1-5
CeilHgt uncert (code) See [1]_, Table 1-6
precipitable_water† Total precipitable water contained in a column of unit cross section from earth to top of atmosphere, cm
Pwat source See [1]_, Table 1-5
Pwat uncert (code) See [1]_, Table 1-6
AOD The broadband aerosol optical depth per unit of air mass due to extinction by aerosol component of atmosphere, unitless
AOD source See [1]_, Table 1-5
AOD uncert (code) See [1]_, Table 1-6
albedo† The ratio of reflected solar irradiance to global horizontal irradiance, unitless
Alb source See [1]_, Table 1-5
Alb uncert (code) See [1]_, Table 1-6
Lprecip depth (mm) The amount of liquid precipitation observed at indicated time for the period indicated in the liquid precipitation quantity field, millimeter
Lprecip quantity (hr) The period of accumulatitudeion for the liquid precipitation depth field, hour
Lprecip source See [1]_, Table 1-5
Lprecip uncert (code) See [1]_, Table 1-6
PresWth (METAR code) Present weather code, see [2]_.
PresWth source Present weather code source, see [2]_.
PresWth uncert (code) Present weather code uncertainty, see [2]_.
======================== ======================================================================================================================================================
.. admonition:: Midnight representation
The function is able to handle midnight represented as 24:00 (NREL TMY3
format, see [1]_) and as 00:00 (SolarAnywhere TMY3 format, see [3]_).
.. warning:: TMY3 irradiance data corresponds to the *previous* hour, so
the first index is 1AM, corresponding to the irradiance from midnight
to 1AM, and the last index is midnight of the *next* year. For example,
if the last index in the TMY3 file was 1988-12-31 24:00:00 this becomes
1989-01-01 00:00:00 after calling :func:`~pvlib.iotools.read_tmy3`.
.. warning:: When coercing the year, the last index in the dataframe will
become midnight of the *next* year. For example, if the last index in
the TMY3 was 1988-12-31 24:00:00, and year is coerced to 1990 then this
becomes 1991-01-01 00:00:00.
References
----------
.. [1] Wilcox, S and Marion, W. "Users Manual for TMY3 Data Sets".
NREL/TP-581-43156, Revised May 2008.
:doi:`10.2172/928611`
.. [2] Wilcox, S. (2007). National Solar Radiation Database 1991 2005
Update: Users Manual. 472 pp.; NREL Report No. TP-581-41364.
:doi:`10.2172/901864`
.. [3] `SolarAnywhere file formats
<https://www.solaranywhere.com/support/historical-data/file-formats/>`_
""" # noqa: E501
head = ['USAF', 'Name', 'State', 'TZ', 'latitude', 'longitude', 'altitude']
with open(str(filename), 'r', encoding=encoding) as fbuf:
# header information on the 1st line (0 indexing)
firstline = fbuf.readline()
# use pandas to read the csv file buffer
# header is actually the second line, but tell pandas to look for
data = pd.read_csv(fbuf, header=0)
meta = dict(zip(head, firstline.rstrip('\n').split(",")))
# convert metadata strings to numeric types
meta['altitude'] = float(meta['altitude'])
meta['latitude'] = float(meta['latitude'])
meta['longitude'] = float(meta['longitude'])
meta['TZ'] = float(meta['TZ'])
meta['USAF'] = int(meta['USAF'])
# get the date column as a pd.Series of numpy datetime64
data_ymd = pd.to_datetime(data['Date (MM/DD/YYYY)'], format='%m/%d/%Y')
# extract minutes
minutes = data['Time (HH:MM)'].str.split(':').str[1].astype(int)
# shift the time column so that midnite is 00:00 instead of 24:00
shifted_hour = data['Time (HH:MM)'].str.split(':').str[0].astype(int) % 24
# shift the dates at midnight (24:00) so they correspond to the next day.
# If midnight is specified as 00:00 do not shift date.
data_ymd[data['Time (HH:MM)'].str[:2] == '24'] += datetime.timedelta(days=1) # noqa: E501
# NOTE: as of pandas>=0.24 the pd.Series.array has a month attribute, but
# in pandas-0.18.1, only DatetimeIndex has month, but indices are immutable
# so we need to continue to work with the panda series of dates `data_ymd`
data_index = pd.DatetimeIndex(data_ymd)
# use indices to check for a leap day and advance it to March 1st
leapday = (data_index.month == 2) & (data_index.day == 29)
data_ymd[leapday] += datetime.timedelta(days=1)
# shifted_hour is a pd.Series, so use pd.to_timedelta to get a pd.Series of
# timedeltas
if coerce_year is not None:
data_ymd = data_ymd.map(lambda dt: dt.replace(year=coerce_year))
data_ymd.iloc[-1] = data_ymd.iloc[-1].replace(year=coerce_year+1)
# NOTE: as of pvlib-0.6.3, min req is pandas-0.18.1, so pd.to_timedelta
# unit must be in (D,h,m,s,ms,us,ns), but pandas>=0.24 allows unit='hour'
data.index = data_ymd + pd.to_timedelta(shifted_hour, unit='h') \
+ pd.to_timedelta(minutes, unit='min')
if map_variables:
data = data.rename(columns=VARIABLE_MAP)
data = data.tz_localize(int(meta['TZ'] * 3600))
return data, meta
[docs]
def read_tmy2(filename):
"""
Read a TMY2 file into a DataFrame.
Note that values contained in the DataFrame are unchanged from the
TMY2 file (i.e. units are retained). Time/Date and location data
imported from the TMY2 file have been modified to a "friendlier"
form conforming to modern conventions (e.g. N latitude is postive, E
longitude is positive, the "24th" hour of any day is technically the
"0th" hour of the next day). In the case of any discrepencies
between this documentation and the TMY2 User's Manual [1]_, the TMY2
User's Manual takes precedence.
Parameters
----------
filename : str
A relative or absolute file path.
Returns
-------
Tuple of the form (data, metadata).
data : DataFrame
A dataframe with the columns described in the table below. For a
more detailed descriptions of each component, please consult the
TMY2 User's Manual [1]_, especially tables 3-1 through 3-6, and
Appendix B.
metadata : dict
The site metadata available in the file.
Notes
-----
The returned structures have the following fields.
============= ==================================
key description
============= ==================================
WBAN Site identifier code (WBAN number)
City Station name
State Station state 2 letter designator
TZ Hours from Greenwich
latitude Latitude in decimal degrees
longitude Longitude in decimal degrees
altitude Site elevation in meters
============= ==================================
============================ ==========================================================================================================================================================================
field description
============================ ==========================================================================================================================================================================
index Pandas timeseries object containing timestamps
year
month
day
hour
ETR Extraterrestrial horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2
ETRN Extraterrestrial normal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2
GHI Direct and diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2
GHISource See [1]_, Table 3-3
GHIUncertainty See [1]_, Table 3-4
DNI Amount of direct normal radiation (modeled) recv'd during 60 mintues prior to timestamp, Wh/m^2
DNISource See [1]_, Table 3-3
DNIUncertainty See [1]_, Table 3-4
DHI Amount of diffuse horizontal radiation recv'd during 60 minutes prior to timestamp, Wh/m^2
DHISource See [1]_, Table 3-3
DHIUncertainty See [1]_, Table 3-4
GHillum Avg. total horizontal illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux (e.g. value of 50 = 5000 lux)
GHillumSource See [1]_, Table 3-3
GHillumUncertainty See [1]_, Table 3-4
DNillum Avg. direct normal illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux
DNillumSource See [1]_, Table 3-3
DNillumUncertainty See [1]_, Table 3-4
DHillum Avg. horizontal diffuse illuminance recv'd during the 60 minutes prior to timestamp, units of 100 lux
DHillumSource See [1]_, Table 3-3
DHillumUncertainty See [1]_, Table 3-4
Zenithlum Avg. luminance at the sky's zenith during the 60 minutes prior to timestamp, units of 10 Cd/m^2 (e.g. value of 700 = 7,000 Cd/m^2)
ZenithlumSource See [1]_, Table 3-3
ZenithlumUncertainty See [1]_, Table 3-4
TotCld Amount of sky dome covered by clouds or obscuring phenonema at time stamp, tenths of sky
TotCldSource See [1]_, Table 3-5
TotCldUncertainty See [1]_, Table 3-6
OpqCld Amount of sky dome covered by clouds or obscuring phenonema that prevent observing the sky at time stamp, tenths of sky
OpqCldSource See [1]_, Table 3-5
OpqCldUncertainty See [1]_, Table 3-6
DryBulb Dry bulb temperature at the time indicated, in tenths of degree C (e.g. 352 = 35.2 C).
DryBulbSource See [1]_, Table 3-5
DryBulbUncertainty See [1]_, Table 3-6
DewPoint Dew-point temperature at the time indicated, in tenths of degree C (e.g. 76 = 7.6 C).
DewPointSource See [1]_, Table 3-5
DewPointUncertainty See [1]_, Table 3-6
RHum Relative humidity at the time indicated, percent
RHumSource See [1]_, Table 3-5
RHumUncertainty See [1]_, Table 3-6
Pressure Station pressure at the time indicated, 1 mbar
PressureSource See [1]_, Table 3-5
PressureUncertainty See [1]_, Table 3-6
Wdir Wind direction at time indicated, degrees from east of north (360 = 0 = north; 90 = East; 0 = undefined,calm)
WdirSource See [1]_, Table 3-5
WdirUncertainty See [1]_, Table 3-6
Wspd Wind speed at the time indicated, in tenths of meters/second (e.g. 212 = 21.2 m/s)
WspdSource See [1]_, Table 3-5
WspdUncertainty See [1]_, Table 3-6
Hvis Distance to discernable remote objects at time indicated (7777=unlimited, 9999=missing data), in tenths of kilometers (e.g. 341 = 34.1 km).
HvisSource See [1]_, Table 3-5
HvisUncertainty See [1]_, Table 3-6
CeilHgt Height of cloud base above local terrain (7777=unlimited, 88888=cirroform, 99999=missing data), in meters
CeilHgtSource See [1]_, Table 3-5
CeilHgtUncertainty See [1]_, Table 3-6
Pwat Total precipitable water contained in a column of unit cross section from Earth to top of atmosphere, in millimeters
PwatSource See [1]_, Table 3-5
PwatUncertainty See [1]_, Table 3-6
AOD The broadband aerosol optical depth (broadband turbidity) in thousandths on the day indicated (e.g. 114 = 0.114)
AODSource See [1]_, Table 3-5
AODUncertainty See [1]_, Table 3-6
SnowDepth Snow depth in centimeters on the day indicated, (999 = missing data).
SnowDepthSource See [1]_, Table 3-5
SnowDepthUncertainty See [1]_, Table 3-6
LastSnowfall Number of days since last snowfall (maximum value of 88, where 88 = 88 or greater days; 99 = missing data)
LastSnowfallSource See [1]_, Table 3-5
LastSnowfallUncertainty See [1]_, Table 3-6
PresentWeather See [1]_, Appendix B. Each string contains 10 numeric values. The string can be parsed to determine each of 10 observed weather metrics.
============================ ==========================================================================================================================================================================
References
----------
.. [1] Marion, W and Urban, K. "Wilcox, S and Marion, W. "User's Manual
for TMY2s". NREL 1995.
:doi:`10.2172/87130`
""" # noqa: E501
# paste in the column info as one long line
string = '%2d%2d%2d%2d%4d%4d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%4d%1s%1d%2d%1s%1d%2d%1s%1d%4d%1s%1d%4d%1s%1d%3d%1s%1d%4d%1s%1d%3d%1s%1d%3d%1s%1d%4d%1s%1d%5d%1s%1d%10d%3d%1s%1d%3d%1s%1d%3d%1s%1d%2d%1s%1d' # noqa: E501
columns = 'year,month,day,hour,ETR,ETRN,GHI,GHISource,GHIUncertainty,DNI,DNISource,DNIUncertainty,DHI,DHISource,DHIUncertainty,GHillum,GHillumSource,GHillumUncertainty,DNillum,DNillumSource,DNillumUncertainty,DHillum,DHillumSource,DHillumUncertainty,Zenithlum,ZenithlumSource,ZenithlumUncertainty,TotCld,TotCldSource,TotCldUncertainty,OpqCld,OpqCldSource,OpqCldUncertainty,DryBulb,DryBulbSource,DryBulbUncertainty,DewPoint,DewPointSource,DewPointUncertainty,RHum,RHumSource,RHumUncertainty,Pressure,PressureSource,PressureUncertainty,Wdir,WdirSource,WdirUncertainty,Wspd,WspdSource,WspdUncertainty,Hvis,HvisSource,HvisUncertainty,CeilHgt,CeilHgtSource,CeilHgtUncertainty,PresentWeather,Pwat,PwatSource,PwatUncertainty,AOD,AODSource,AODUncertainty,SnowDepth,SnowDepthSource,SnowDepthUncertainty,LastSnowfall,LastSnowfallSource,LastSnowfallUncertaint' # noqa: E501
hdr_columns = 'WBAN,City,State,TZ,latitude,longitude,altitude'
tmy2, tmy2_meta = _read_tmy2(string, columns, hdr_columns, str(filename))
return tmy2, tmy2_meta
def _parsemeta_tmy2(columns, line):
"""Retrieve metadata from the top line of the tmy2 file.
Parameters
----------
columns : string
String of column headings in the header
line : string
Header string containing DataFrame
Returns
-------
meta : Dict of metadata contained in the header string
"""
# Remove duplicated spaces, and read in each element
rawmeta = " ".join(line.split()).split(" ")
meta = rawmeta[:3] # take the first string entries
meta.append(int(rawmeta[3]))
# Convert to decimal notation with S negative
longitude = (
float(rawmeta[5]) + float(rawmeta[6])/60) * (2*(rawmeta[4] == 'N') - 1)
# Convert to decimal notation with W negative
latitude = (
float(rawmeta[8]) + float(rawmeta[9])/60) * (2*(rawmeta[7] == 'E') - 1)
meta.append(longitude)
meta.append(latitude)
meta.append(float(rawmeta[10]))
# Creates a dictionary of metadata
meta_dict = dict(zip(columns.split(','), meta))
return meta_dict
def _read_tmy2(string, columns, hdr_columns, fname):
head = 1
date = []
with open(fname) as infile:
fline = 0
for line in infile:
# Skip the header
if head != 0:
meta = _parsemeta_tmy2(hdr_columns, line)
head -= 1
continue
# Reset the cursor and array for each line
cursor = 1
part = []
for marker in string.split('%'):
# Skip the first line of markers
if marker == '':
continue
# Read the next increment from the marker list
increment = int(re.findall(r'\d+', marker)[0])
next_cursor = cursor + increment
# Extract the value from the line in the file
val = (line[cursor:next_cursor])
# increment the cursor by the length of the read value
cursor = next_cursor
# Determine the datatype from the marker string
if marker[-1] == 'd':
try:
val = float(val)
except ValueError:
raise ValueError('WARNING: In {} Read value is not an '
'integer " {} " '.format(fname, val))
elif marker[-1] == 's':
try:
val = str(val)
except ValueError:
raise ValueError('WARNING: In {} Read value is not a '
'string " {} " '.format(fname, val))
else:
raise Exception('WARNING: In {} Improper column DataFrame '
'" %{} " '.format(__name__, marker))
part.append(val)
if fline == 0:
axes = [part]
year = part[0] + 1900
fline = 1
else:
axes.append(part)
# Create datetime objects from read data
date.append(datetime.datetime(year=int(year),
month=int(part[1]),
day=int(part[2]),
hour=(int(part[3]) - 1)))
data = pd.DataFrame(
axes, index=date,
columns=columns.split(',')).tz_localize(int(meta['TZ'] * 3600))
return data, meta
