diff --git a/DMI_Open_Data_dialog.py b/DMI_Open_Data_dialog.py index ac2a765..6315c54 100644 --- a/DMI_Open_Data_dialog.py +++ b/DMI_Open_Data_dialog.py @@ -2,6 +2,7 @@ import os from typing import Tuple, Dict, Set +from datetime import datetime as dt from qgis.PyQt import QtWidgets, uic import requests import pandas as pd @@ -15,9 +16,12 @@ from PyQt5.QtWidgets import * from qgis.PyQt.QtCore import QVariant import webbrowser + +from .util import rfc3339_zulu_format from .forecast_para import depth_para_dkss, salinity_nsbs, salinity_idw, salinity_if, salinity_lb, salinity_lf, salinity_ws, water_temp_nsbs, water_temp_if, water_temp_lb, water_temp_lf, water_temp_ws, water_temp_idw, v_current_nsbs, v_current_idw, v_current_if, v_current_lb, v_current_lf, v_current_ws, u_current_nsbs, u_current_idw, u_current_if,u_current_lb, u_current_lf, u_current_ws import processing -from .api.station import get_stations, StationApi, StationId, Station, Parameter +from .api.station import get_folded_stations, get_stations, StationApi, StationId, Station, Parameter, StationCountry, \ + StationOwner from .settings import DMISettingsManager, DMISettingKeys warnings.simplefilter(action='ignore', category=FutureWarning) @@ -241,7 +245,7 @@ def get_stations_and_parameters_if_settings_allow(self, station_type: StationApi stations = [] parameters = {} if api_key: - stations = get_stations(station_type, api_key) + stations = get_folded_stations(station_type, api_key) parameters = {parameter for station in stations.values() for parameter in station.parameters } return stations, parameters @@ -1148,100 +1152,83 @@ def run(self): project = QgsProject.instance() project.addMapLayer(layer, addToLegend=False) layer_group.insertLayer(-1, layer) -# Information about stations and parameters + # Information about stations and parameters if dataName == 'Stations and Parameters': if self.met_stat_info.isChecked(): - data_type = 'climateData' + station_api = StationApi.CLIMATE_STATION_VALUE api_key = self.settings_manager.value(DMISettingKeys.CLIMATEDATA_API_KEY.value) - data_type2 = 'station' elif self.tide_info.isChecked(): - data_type = 'oceanObs' + station_api = StationApi.OCEAN_OBS api_key = self.settings_manager.value(DMISettingKeys.OCEANOBS_API_KEY.value) - data_type2 = 'station' - url = 'https://dmigw.govcloud.dk/v2/' + data_type + '/collections/' + data_type2 +'/items' - params = {'api-key': api_key} -# metObs info + + name_stations_met = '' + status = None + start = None + end = None + country: StationCountry = None + owner: StationOwner = None + + # metObs info if self.met_stat_info.isChecked(): - if self.radioButton_11.isChecked() and self.radioButton.isChecked(): - params.update({'datetime': datetime, - 'status': 'Active'}) - elif self.radioButton_10.isChecked() and self.radioButton_9.isChecked(): - params = params - elif self.radioButton.isChecked() and self.radioButton_10.isChecked(): - params.update({'status': 'Active'}) - elif self.radioButton_9.isChecked() and self.radioButton_11.isChecked(): - params.update({'datetime': datetime}) -# ocean info + if self.radioButton.isChecked(): + status = 'Active' + + if self.radioButton_11.isChecked(): + start = dt.strptime(start_datetime, rfc3339_zulu_format) + end = dt.strptime(end_datetime, rfc3339_zulu_format) + + if self.radioButton_2.isChecked(): + country = StationCountry.DENMARK + name_stations_met = 'Meteorological Stations Denmark' + elif self.radioButton_3.isChecked(): + country = StationCountry.GREENLAND + name_stations_met = 'Meteorological Stations Greenland' + elif self.radioButton_4.isChecked(): + name_stations_met = 'All Meteorological Stations' + + # ocean info if self.tide_info.isChecked(): - if self.radioButton_20.isChecked() and self.radioButton_22.isChecked(): - params.update({'datetime': datetime, - 'status': 'Active'}) - elif self.radioButton_19.isChecked() and self.radioButton_21.isChecked(): - params = params - elif self.radioButton_20.isChecked() and self.radioButton_21.isChecked(): - params.update({'status': 'Active'}) - elif self.radioButton_19.isChecked() and self.radioButton_22.isChecked(): - params.update({'datetime': datetime}) - r = requests.get(url, params=params) - print(r.url) - json = r.json() - r_code = r.status_code - if r_code == 403: - QMessageBox.warning(self, self.tr("DMI Open Data"), - self.tr('API Key is not valid or is expired / revoked.')) - else: - df = json_normalize(json['features']) - # Name and sort the data based on users preferences - if self.met_stat_info.isChecked(): - if self.radioButton_2.isChecked(): - df = df.loc[df['properties.country'] == 'DNK'] - name_stations_met = 'Meteorological Stations Denmark' - elif self.radioButton_3.isChecked(): - df = df.loc[df['properties.country'] == 'GRL'] - name_stations_met = 'Meteorological Stations Greenland' - elif self.radioButton_4.isChecked(): - name_stations_met = 'All Meteorological Stations' - if self.tide_info.isChecked(): - if self.radioButton_14.isChecked(): - name_stations_met = 'All stations' - elif self.radioButton_12.isChecked(): - df = df.loc[df['properties.owner'] == 'DMI'] - name_stations_met = 'DMI' - elif self.radioButton_13.isChecked(): - df = df.loc[df['properties.owner'] == 'Kystdirektoratet / Coastal Authority'] - name_stations_met = 'Coastal Authority' - # Names the layer as the station type and parameter if parameter is chosen. - if len(parameters) != 0: - df = df[pd.DataFrame(df['properties.parameterId'].tolist()).isin(parameters).any(1).values] - name_stations_met = name_stations_met + ' ' + parameters[0] - if len(df) == 0: + if self.radioButton_20.isChecked(): + status = 'Active' + + if self.radioButton_22.isChecked(): + start = dt.strptime(start_datetime, rfc3339_zulu_format) + end = dt.strptime(end_datetime, rfc3339_zulu_format) + + if self.radioButton_14.isChecked(): + name_stations_met = 'All stations' + elif self.radioButton_12.isChecked(): + owner = StationOwner.DMI + name_stations_met = 'DMI' + elif self.radioButton_13.isChecked(): + owner = StationOwner.KDI + name_stations_met = 'Coastal Authority' + + # Names the layer as the station type and parameter if parameter is chosen. + if len(parameters) != 0: + name_stations_met = name_stations_met + ' ' + parameters[0] + + try: + stations = list(get_stations(station_api, api_key, status=status, start_datetime=start, end_datetime=end, + country=country, owner=owner, parameters=parameters)) + if len(stations) == 0: QMessageBox.warning(self, self.tr("DMI Open Data"), self.tr('No stations meets this requirement.')) - elif len(df) > 0: - # QGIS geometry - vl = QgsVectorLayer("Point", name_stations_met, "memory") - for row in df.itertuples(): - pr = vl.dataProvider() - vl.startEditing() - for head in df: - if head != 'geometry.coordinates': - pr.addAttributes([QgsField(head, QVariant.String)]) - vl.updateFields() - f = QgsFeature() - f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(row[3][0], row[3][1]))) - if data_type == 'climateData': - f.setAttributes([row[1],row[2],row[4],row[5],row[6],row[7],\ - row[8],row[9],row[10],row[11],str(row[12]),row[13],row[14],\ - row[15],row[16],row[17],row[18],row[19],row[20],row[21],row[22]]) - elif data_type == 'oceanObs': - f.setAttributes([row[1], row[2], row[4], row[5], row[6], str(row[7]), \ - row[8], row[9], row[10], row[11], str(row[12]), row[13], row[14], \ - row[15], row[16], row[17], row[18], row[19], row[20], row[21]]) - vl.addFeature(f) - vl.updateExtents() - vl.commitChanges() - QgsProject.instance().addMapLayer(vl) - iface.zoomToActiveLayer() + vl = QgsVectorLayer("Point", name_stations_met, "memory") + pr = vl.dataProvider() + vl.startEditing() + pr.addAttributes(Station.qgs_fields().toList()) + vl.updateFields() + for station in stations: + vl.addFeature(station.as_qgs_feature()) + vl.updateExtents() + vl.commitChanges() + QgsProject.instance().addMapLayer(vl) + except Exception as ex: + QMessageBox.warning(self, self.tr("DMI Open Data"), + self.tr(str(ex))) + return # No point in continuing... + iface.zoomToActiveLayer() if len(error_stats) != 0: QMessageBox.warning(self, self.tr("DMI Open Data"), - self.tr('Following stations does not produce data.' + '\n' + 'Change parameters, time and/or resolution.' + '\n' + '\n' + '\n'.join(error_stats))) \ No newline at end of file + self.tr('Following stations does not produce data.' + '\n' + 'Change parameters, time and/or resolution.' + '\n' + '\n' + '\n'.join(error_stats))) diff --git a/api/station.py b/api/station.py index dcc81ce..a33f1e5 100644 --- a/api/station.py +++ b/api/station.py @@ -1,14 +1,30 @@ +from datetime import datetime from itertools import groupby -from typing import Tuple, Dict, List - +from typing import Dict, List, Iterable +from functools import lru_cache +from PyQt5.QtCore import QVariant +from qgis.core import QgsFeature, QgsFields, QgsField, QgsGeometry, QgsPointXY import requests from enum import Enum +from operator import attrgetter +from ..util import rfc3339_zulu_format StationId = str StationName = str Parameter = str +def get_qvariant(python_type): + if python_type == str: + return QVariant.String + if python_type == float: + return QVariant.Double + if python_type == int: + return QVariant.Int + if python_type == datetime: + return QVariant.DateTime + + class StationApi(Enum): class _Inner: base_url_path: str @@ -28,15 +44,91 @@ def get_api_name(self) -> str: return 'Climate Data API' +class StationStatus(Enum): + ACTIVE = 'Active' + INACTIVE = 'Inactive' + + +class StationOwner(Enum): + DMI = 'DMI' + KDI = 'Kystdirektoratet / Coastal Authority' + + +class StationCountry(Enum): + DENMARK = 'DNK' + GREENLAND = 'GRL' + + class Station: + latitude: float + longitude: float station_id: str station_name: str parameters: List[Parameter] - - def __init__(self, station_id, station_name, parameters): - self.station_id = station_id - self.station_name = station_name - self.parameters = parameters + barometer_height: float + country: str + created: datetime + operation_from: datetime + operation_to: datetime + owner: str + region_id: str + station_height: float + status: str + station_type: str + updated: str + valid_from: datetime + valid_to: datetime + wmo_country_code: str + wmo_station_id: str + + def __init__(self, geojson_feature): + self.longitude = geojson_feature['geometry']['coordinates'][0] + self.latitude = geojson_feature['geometry']['coordinates'][1] + self.station_id = geojson_feature['properties']['stationId'] + self.station_name = geojson_feature['properties']['name'] + self.parameters = geojson_feature['properties']['parameterId'] + # Some stations (like oceanObs stations) does not have barometer height, therefore this attribute is optional + if barometer_height := geojson_feature['properties'].get('barometerHeight'): + self.barometer_height = barometer_height + self.country = geojson_feature['properties']['country'] + self.created = datetime.strptime(geojson_feature['properties']['created'], rfc3339_zulu_format) + if operation_from := geojson_feature['properties']['operationFrom']: + self.operation_from = datetime.strptime(operation_from, rfc3339_zulu_format) + if operation_to := geojson_feature['properties']['operationTo']: + self.operation_to = datetime.strptime(operation_to, rfc3339_zulu_format) + self.owner = geojson_feature['properties']['owner'] + self.region_id = geojson_feature['properties']['regionId'] + if station_height := geojson_feature['properties'].get('stationHeight'): + self.station_height = station_height + self.status = geojson_feature['properties']['status'] + self.station_type = geojson_feature['properties']['type'] + self.updated = geojson_feature['properties']['updated'] + self.valid_from = datetime.strptime(geojson_feature['properties']['validFrom'], rfc3339_zulu_format) + if valid_to := geojson_feature['properties']['validTo']: + self.valid_to = datetime.strptime(valid_to, rfc3339_zulu_format) + self.wmo_country_code = geojson_feature['properties']['wmoCountryCode'] + self.wmo_station_id = geojson_feature['properties']['wmoStationId'] + + def as_qgs_feature(self) -> QgsFeature: + qgs_feature = QgsFeature() + qgs_feature.setFields(Station.qgs_fields()) + qgs_feature.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(self.longitude, self.latitude))) + for attr, value in [(attr, value) for attr, value in vars(self).items() if attr not in ['longitude', 'latitude', 'parameters']]: + # While QGIS accepts QVariant.Datetime as QgsField type (even though the documentation says otherwise) the + # value of the field still has to be a string. With ISO formatting the temporal features of QGIS works when + # the field type is QVariant.Datetime + if type(value) == datetime: + value = value.isoformat() + qgs_feature.setAttribute(attr, value) + return qgs_feature + + @classmethod + @lru_cache(maxsize=None) + def qgs_fields(cls) -> QgsFields: + qgs_fields = QgsFields() + for property_name, python_type in [(p, t) for p, t in cls.__annotations__.items() if p not in ['latitude', 'longitude', 'parameters']]: + qgs_fields.append(QgsField(property_name, get_qvariant(python_type))) + return qgs_fields class StationAPIGenericException(Exception): @@ -49,15 +141,25 @@ def __init__(self, station_api: StationApi): super().__init__(f"API Key for {station_api.get_api_name()} is not valid or is expired / revoked.") -def get_stations(station_api: StationApi, api_key: str) -> Dict[StationId, Station]: - """ - Generates station ids and corresponding names, picking current name for active stations, and latest used name for - inactive ones - :param station_api: ObsApi enum of which API should be used - :param api_key: API key for calls to the API - :return: dictionary of station id and name - """ +def _station_api_call(station_api: StationApi, api_key: str, status: StationStatus = None, + start_datetime: datetime = None, end_datetime: datetime = None, station_type: str = None) -> Iterable[Station]: station_params = {'api-key': api_key} + if status: + station_params['status'] = status.value + if start_datetime or end_datetime: + datetime_param = '' + if start_datetime: + datetime_param += start_datetime.strftime(rfc3339_zulu_format) + else: + datetime_param += '..' + datetime_param += '/' + if end_datetime: + datetime_param += end_datetime.strftime(rfc3339_zulu_format) + else: + datetime_param += '..' + station_params['datetime'] = datetime_param + if station_type: + station_params['type'] = station_type try: obs_station_request = requests.get( f'https://dmigw.govcloud.dk/{station_api.value.base_url_path}/collections/station/items', @@ -72,9 +174,37 @@ def get_stations(station_api: StationApi, api_key: str) -> Dict[StationId, Stati raise StationAPIGenericException() json = obs_station_request.json() + return map(lambda station_json_feature: Station(station_json_feature), json['features']) + + +def get_folded_stations(station_api: StationApi, api_key: str) -> Dict[StationId, Station]: + """ + Generates station ids and corresponding names, picking current name for active stations, and latest used name for + inactive ones + :param station_api: ObsApi enum of which API should be used + :param api_key: API key for calls to the API + :return: dictionary of station id and name + """ + stations = _station_api_call(station_api, api_key) station_map = {} - for station_id, stations in groupby(json['features'], key=lambda station: station['properties']['stationId']): + stations = sorted(stations, key=attrgetter('station_id')) # pre-sort for groupby to work correctly + for station_id, station_group in groupby(stations, key=attrgetter('station_id')): # Choose the latest station record to pick station name from - latest_station = sorted(stations, key=lambda station: station['properties']['validFrom'])[-1] - station_map[station_id] = Station(station_id, latest_station['properties']['name'], latest_station['properties']['parameterId']) + latest_station = sorted(station_group, key=attrgetter('valid_from'))[-1] + station_map[station_id] = latest_station return station_map + + +def get_stations(station_api: StationApi, api_key: str, status=None, start_datetime: datetime = None, + end_datetime: datetime = None, country: StationCountry = None, owner: StationOwner = None, + station_type: str = None, + parameters: List[str] = None) -> Iterable[Station]: + stations = _station_api_call(station_api, api_key, status=status, start_datetime=start_datetime, + end_datetime=end_datetime, station_type=station_type) + if country: + stations = filter(lambda station: station.country == country.value, stations) + if owner: + stations = filter(lambda station: station.owner == owner.value, stations) + if parameters: + stations = filter(lambda station: any(parameter in station.parameters for parameter in parameters), stations) + return stations diff --git a/pb_tool.cfg b/pb_tool.cfg index 214bb78..ca14b03 100644 --- a/pb_tool.cfg +++ b/pb_tool.cfg @@ -43,7 +43,7 @@ extras: metadata.txt icon.png # Other directories to be deployed with the plugin. # These must be subdirectories under the plugin directory -extra_dirs: api settings +extra_dirs: api settings util # ISO code(s) for any locales (translations), separated by spaces. # Corresponding .ts files must exist in the i18n directory diff --git a/util/__init__.py b/util/__init__.py new file mode 100644 index 0000000..c70cf38 --- /dev/null +++ b/util/__init__.py @@ -0,0 +1,2 @@ + +rfc3339_zulu_format = '%Y-%m-%dT%H:%M:%SZ'