GDALSource WCS support

src/CMakeLists.txt

@@ -233,9 +233,11 @@ find_package(Boost COMPONENTS date_time filesystem REQUIRED)
 target_link_libraries(mapping_core_base_lib Boost::date_time Boost::filesystem)
 target_include_directories(mapping_core_base_lib PRIVATE ${Boost_INCLUDE_DIRS})
 
-find_package(GDAL REQUIRED)
-target_link_libraries(mapping_core_base_lib ${GDAL_LIBRARY})
-target_include_directories(mapping_core_base_lib PUBLIC ${GDAL_INCLUDE_DIR})
+target_link_libraries(mapping_core_base_lib /usr/local/lib/libgdal.so) #linking self build GDAL 2.3
+target_include_directories(mapping_core_base_lib PRIVATE /usr/local/include)
+#find_package(GDAL REQUIRED)
+#target_link_libraries(mapping_core_base_lib ${GDAL_LIBRARY})
+#target_include_directories(mapping_core_base_lib PUBLIC ${GDAL_INCLUDE_DIR})
 
 find_package(PNG REQUIRED)
 target_link_libraries(mapping_core_base_lib PNG::PNG)

src/operators/source/gdal_source.cpp

@@ -85,8 +85,7 @@ void RasterGDALSourceOperator::writeSemanticParameters(std::ostringstream &strea
 
 // load the json definition of the dataset, then get the file to be loaded from GDALTimesnap. Finally load the raster.
 std::unique_ptr<GenericRaster> RasterGDALSourceOperator::getRaster(const QueryRectangle &rect, const QueryTools &tools) {
-	Json::Value datasetJson = GDALSourceDataSets::getDataSetDescription(sourcename);
-	GDALTimesnap::GDALDataLoadingInfo loadingInfo = GDALTimesnap::getDataLoadingInfo(datasetJson, channel, rect);
+	GDALTimesnap::GDALDataLoadingInfo loadingInfo = GDALTimesnap::getDataLoadingInfo(sourcename, channel, rect);
 	auto raster = loadDataset(loadingInfo, rect.crsId, true, rect);
 	//flip here so the tiff result will not be flipped
 	return raster->flip(false, true);
@@ -268,7 +267,17 @@ std::unique_ptr<GenericRaster> RasterGDALSourceOperator::loadDataset(const GDALT
 
 	GDAL::init();
 
-	auto dataset = (GDALDataset *) GDALOpen(loadingInfo.fileName.c_str(), GA_ReadOnly);
+    const bool isWCS = loadingInfo.fileName.find("WCS:") == 0;
+    GDALDataset *dataset = nullptr;
+
+    if(isWCS){
+        //WCS requests seem to need NoGridAxisSwap set to true, else the request throws an error.
+        std::string opt = "NoGridAxisSwap=true";
+        const char *const strs[] = {opt.c_str(), nullptr };
+        dataset = (GDALDataset *) GDALOpenEx(loadingInfo.fileName.c_str(), GDAL_OF_RASTER, nullptr, strs, nullptr);
+    } else {
+        dataset = (GDALDataset *) GDALOpen(loadingInfo.fileName.c_str(), GA_ReadOnly);
+    }
 
 	if (dataset == nullptr)
 		throw OperatorException(concat("GDAL Source: Could not open dataset ", loadingInfo.fileName));

src/operators/source/ogr_raw_source.cpp

@@ -1,4 +1,4 @@
-#include <gdal/ogrsf_frmts.h>
+#include <ogrsf_frmts.h>
 #include "operators/operator.h"
 #include "datatypes/polygoncollection.h"
 #include "datatypes/linecollection.h"

src/util/gdal_timesnap.cpp

@@ -3,6 +3,8 @@
 
 constexpr int MAX_FILE_NAME_LENGTH = 255;
 
+const std::string GDALTimesnap::placeholder = "%%%TIME_STRING%%%";
+
 const std::map<std::string, TimeUnit> GDALTimesnap::string_to_TimeUnit = {
         {"Second", 	TimeUnit::Second},
         {"Minute", 	TimeUnit::Minute},
@@ -106,8 +108,10 @@ ptime GDALTimesnap::snapToInterval(TimeUnit snapUnit, int intervalValue, ptime s
 // calculates the filename for queried time by snapping the wanted time to the 
 // nearest smaller timestamp that exists for the dataset
 // TODO: move general parameter parsing to a more appropriate function
-GDALTimesnap::GDALDataLoadingInfo GDALTimesnap::getDataLoadingInfo(Json::Value datasetJson, int channel, const TemporalReference &tref)
+GDALTimesnap::GDALDataLoadingInfo GDALTimesnap::getDataLoadingInfo(const std::string &sourcename, int channel, const TemporalReference &tref)
 {
+    Json::Value datasetJson = GDALSourceDataSets::getDataSetDescription(sourcename);
+
     // get parameters
     Json::Value channelJson = datasetJson["channels"][channel];
 
@@ -126,79 +130,82 @@ GDALTimesnap::GDALDataLoadingInfo GDALTimesnap::getDataLoadingInfo(Json::Value d
     double time_start_mapping;
     double time_end_mapping;
 
-    if(time_start.empty()) {
+    if (time_start.empty()) {
         time_start_mapping = tref.beginning_of_time();
     } else {
         time_start_mapping = timeParser->parse(time_start);
     }
 
-    if(time_end.empty()) {
+    if (time_end.empty()) {
         time_end_mapping = tref.end_of_time();
     } else {
         time_end_mapping = timeParser->parse(time_end);
     }
 
-    double wantedTimeUnix = tref.t1;
-
-    //check if requested time is in range of dataset timestamps
-    //a dataset only has start not end time. if wantedtime is past the last file of dataset, it can simply not be loaded.
-    if(wantedTimeUnix < time_start_mapping || wantedTimeUnix > time_end_mapping)
-        throw NoRasterForGivenTimeException("Requested time is not in range of dataset");
-
-    if(datasetJson.isMember("time_interval") || channelJson.isMember("time_interval")) {
-        Json::Value timeInterval = channelJson.get("time_interval", datasetJson.get("time_interval", Json::Value(Json::objectValue)));
-        TimeUnit intervalUnit 	 = GDALTimesnap::createTimeUnit(timeInterval.get("unit", "Month").asString());
-        int intervalValue 		 = timeInterval.get("value", 1).asInt();
-
-
-		ptime start = from_time_t(static_cast<time_t>(time_start_mapping));
-        ptime wanted = from_time_t(static_cast<time_t>(wantedTimeUnix));
-
-		//snap the time to the given interval
-        ptime snappedTimeStart = GDALTimesnap::snapToInterval(intervalUnit, intervalValue, start, wanted);
-
-        // snap end to the next interval
-		ptime snappedTimeEnd(snappedTimeStart);
-		switch (intervalUnit) {
-			case TimeUnit::Year:
-				snappedTimeEnd += boost::gregorian::years(intervalValue);
-                break;
-			case TimeUnit::Month:
-				snappedTimeEnd += boost::gregorian::months(intervalValue);
-                break;
-			case TimeUnit::Day:
-				snappedTimeEnd += boost::gregorian::days(intervalValue);
-                break;
-			case TimeUnit::Hour:
-				snappedTimeEnd += boost::posix_time::hours(intervalValue);
-                break;
-			case TimeUnit::Minute:
-				snappedTimeEnd += boost::posix_time::minutes(intervalValue);
-                break;
-			case TimeUnit::Second:
-				snappedTimeEnd += boost::posix_time::seconds(intervalValue);
-                break;
-		}
-
-        time_start_mapping = boost::posix_time::to_time_t(snappedTimeStart);
-        time_end_mapping = boost::posix_time::to_time_t(snappedTimeEnd);
-
-        // format date to determine file
-        auto snappedTimeT = static_cast<time_t>(time_start_mapping);
-        tm snappedTimeTm = {};
-        gmtime_r(&snappedTimeT, &snappedTimeTm);
-
-        char date[MAX_FILE_NAME_LENGTH] = {0};
-        strftime(date, sizeof(date), time_format.c_str(), &snappedTimeTm);
-        std::string snappedTimeString(date);
-
-        std::string placeholder = "%%%TIME_STRING%%%";
-        size_t placeholderPos = fileName.find(placeholder);
-
-        fileName = fileName.replace(placeholderPos, placeholder.length(), snappedTimeString);
+    const bool has_placeholder = fileName.find(placeholder) != std::string::npos;
+
+    if(has_placeholder) {
+        double wantedTimeUnix = tref.t1;
+
+        //check if requested time is in range of dataset timestamps
+        //a dataset only has start not end time. if wantedtime is past the last file of dataset, it can simply not be loaded.
+        if (wantedTimeUnix < time_start_mapping || wantedTimeUnix > time_end_mapping)
+            throw NoRasterForGivenTimeException("Requested time is not in range of dataset");
+
+        if (datasetJson.isMember("time_interval") || channelJson.isMember("time_interval")) {
+            Json::Value timeInterval = channelJson.get("time_interval", datasetJson.get("time_interval", Json::Value(
+                    Json::objectValue)));
+            TimeUnit intervalUnit = GDALTimesnap::createTimeUnit(timeInterval.get("unit", "Month").asString());
+            int intervalValue = timeInterval.get("value", 1).asInt();
+
+
+            ptime start = from_time_t(static_cast<time_t>(time_start_mapping));
+            ptime wanted = from_time_t(static_cast<time_t>(wantedTimeUnix));
+
+            //snap the time to the given interval
+            ptime snappedTimeStart = GDALTimesnap::snapToInterval(intervalUnit, intervalValue, start, wanted);
+
+            // snap end to the next interval
+            ptime snappedTimeEnd(snappedTimeStart);
+            switch (intervalUnit) {
+                case TimeUnit::Year:
+                    snappedTimeEnd += boost::gregorian::years(intervalValue);
+                    break;
+                case TimeUnit::Month:
+                    snappedTimeEnd += boost::gregorian::months(intervalValue);
+                    break;
+                case TimeUnit::Day:
+                    snappedTimeEnd += boost::gregorian::days(intervalValue);
+                    break;
+                case TimeUnit::Hour:
+                    snappedTimeEnd += boost::posix_time::hours(intervalValue);
+                    break;
+                case TimeUnit::Minute:
+                    snappedTimeEnd += boost::posix_time::minutes(intervalValue);
+                    break;
+                case TimeUnit::Second:
+                    snappedTimeEnd += boost::posix_time::seconds(intervalValue);
+                    break;
+            }
+
+            time_start_mapping = boost::posix_time::to_time_t(snappedTimeStart);
+            time_end_mapping = boost::posix_time::to_time_t(snappedTimeEnd);
+
+            // format date to determine file
+            auto snappedTimeT = static_cast<time_t>(time_start_mapping);
+            tm snappedTimeTm = {};
+            gmtime_r(&snappedTimeT, &snappedTimeTm);
+
+            char date[MAX_FILE_NAME_LENGTH] = {0};
+            strftime(date, sizeof(date), time_format.c_str(), &snappedTimeTm);
+            std::string snappedTimeString(date);
+
+            size_t placeholderPos = fileName.find(placeholder);
+
+            fileName = fileName.replace(placeholderPos, placeholder.length(), snappedTimeString);
+        }
     }
 
-
     // other GDAL parameters
     Unit unit = Unit::unknown();
     if (channelJson.isMember("unit")) {
@@ -213,10 +220,15 @@ GDALTimesnap::GDALDataLoadingInfo GDALTimesnap::getDataLoadingInfo(Json::Value d
     auto coords = channelJson.get("coords", datasetJson["coords"]);
     CrsId crsId = CrsId::from_srs_string(coords.get("crs", "").asString());
 
-
-    boost::filesystem::path file_path(path);
-    file_path /= fileName;
-	return GDALDataLoadingInfo(file_path.string(), channel,
-                               TemporalReference(TIMETYPE_UNIX, time_start_mapping, time_end_mapping),
-                               crsId, nodata, unit);
+    if(path.empty()) {
+        return GDALDataLoadingInfo(fileName, channel,
+                                   TemporalReference(TIMETYPE_UNIX, time_start_mapping, time_end_mapping),
+                                   std::move(crsId), nodata, unit);
+    } else {
+        boost::filesystem::path file_path(path);
+        file_path /= fileName;
+        return GDALDataLoadingInfo(file_path.string(), channel,
+                                   TemporalReference(TIMETYPE_UNIX, time_start_mapping, time_end_mapping),
+                                   std::move(crsId), nodata, unit);
+    }
 }

src/util/gdal_timesnap.h

@@ -11,6 +11,7 @@
 #include "util/timeparser.h"
 #include "datatypes/spatiotemporal.h"
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include "util/gdal_source_datasets.h"
 
 using namespace boost::posix_time;
 
@@ -34,7 +35,7 @@ class GDALTimesnap {
 	public:
 		class GDALDataLoadingInfo {
 		public:
-			GDALDataLoadingInfo(std::string fileName, int channel, const TemporalReference &tref, const CrsId &crsId,
+			GDALDataLoadingInfo(const std::string &fileName, int channel, const TemporalReference &tref, CrsId &&crsId,
                                 double nodata, const Unit &unit):
 					fileName(std::move(fileName)), channel(channel), tref(tref), crsId(crsId), nodata(nodata), unit(unit) {}
 
@@ -49,10 +50,11 @@ class GDALTimesnap {
 
         static ptime snapToInterval(TimeUnit unit, int unitValue, ptime startTime, ptime wantedTime);
 
-		static GDALDataLoadingInfo getDataLoadingInfo(Json::Value datasetJson, int channel, const TemporalReference &tref);
+		static GDALDataLoadingInfo getDataLoadingInfo(const std::string &sourcename, int channel, const TemporalReference &tref);
 
 		static TimeUnit createTimeUnit(std::string value);
 		static const std::map<std::string, TimeUnit> string_to_TimeUnit;
+		static const std::string placeholder;
 };
 
 #endif

src/util/ogr_source_datasets.h

@@ -3,7 +3,7 @@
 #define MAPPING_CORE_OGR_SOURCE_DATASETS_H
 
 #include <json/json.h>
-#include <gdal/ogrsf_frmts.h>
+#include <ogrsf_frmts.h>
 
 /**
  * This class provides information about available OGR datasets that can be opened with the OGRSourceOperator.

src/util/ogr_source_util.cpp

@@ -1,4 +1,4 @@
-#include <gdal/ogrsf_frmts.h>
+#include <ogrsf_frmts.h>
 #include "operators/operator.h"
 #include "datatypes/polygoncollection.h"
 #include "datatypes/linecollection.h"

src/util/ogr_source_util.h

@@ -1,7 +1,7 @@
 #ifndef UTIL_OGR_SOURCE_UTIL_H
 #define UTIL_OGR_SOURCE_UTIL_H
 
-#include <gdal/ogrsf_frmts.h>
+#include <ogrsf_frmts.h>
 #include "util/exceptions.h"
 #include "util/enumconverter.h"
 #include "util/timeparser.h"

test/systemtests/data/gdal_files/test_no_snapping.json

@@ -0,0 +1,32 @@
+{
+	"dataset_name" : "test_no_snapping",
+	"path" : "../../test/systemtests/data/gdal_files/TestHour/",	
+
+	"channels" : [
+		{
+
+		},
+		{
+			"coords" : {
+				"crs" : "EPSG:4326"
+			},
+			"file_name" : "MOD13A2_M_NDVI_2014-09-01T17_00_00.TIFF",
+			"time_start" : "2014-09-01T17:00:00",
+			"time_end" : "2014-09-01T20:00:00",
+			"channel" : 1,
+			"description" : "early good data"
+		},
+		{
+
+			"coords" : {
+				"crs" : "EPSG:4326"
+			},
+			"file_name" : "MOD13A2_M_NDVI_2014-09-01T23_00_00.TIFF",
+			"time_start" : "2014-09-01T23:00:00",
+			"time_end" : "2014-09-02T02:00:00",			
+			"channel" : 1,
+			"description" : "good data, a little later"
+		}
+	],	
+	"description" : "good data"
+}

test/systemtests/data/gdal_files/test_wcs.json

@@ -0,0 +1,25 @@
+{
+	"dataset_name" : "test_wcs",
+	"path" : "",
+
+	"channels" : [
+		{
+
+		},		
+		{
+			"coords" : {
+				"crs" : "EPSG:4326"
+			},
+			"file_name" : "WCS:https://demo.geo-solutions.it/geoserver/ows?version=2.0.1&coverage=geosolutions:NaturalEarth",
+			"channel" : 1
+		},
+		{
+			"coords" : {
+				"crs" : "EPSG:4326"
+			},			
+			"file_name" : "WCS:https://maps.dwd.de/geoserver/ows?version=2.0.1&coverage=dwd:GRAY_HR_SR_OB_DR",
+			"channel" : 1
+		}
+	],	
+	"description" : "test wcs requests"
+}

test/systemtests/queries/gdal_source_no_snapping.json

@@ -0,0 +1,29 @@
+{
+    "name": "Load world raster",
+	"query_result": "raster",
+    "temporal_reference": {
+        "type": "UNIX",
+        "start": 1409596200
+    },
+    "spatial_reference": {
+        "projection": "EPSG:4326",
+        "x1": -160,
+        "x2": 160,
+        "y1": -70,
+        "y2": 70
+    },
+    "resolution": {
+    	"type": "pixels",
+        "x": 3600,
+        "y": 1800
+    },
+    "query_mode": "exact",
+    "query": {        
+        "params": {
+       		"channel": 3,
+            "sourcename": "test_no_snapping"
+        },
+        "type": "gdal_source"
+    },
+    "query_expected_hash": ""
+}