Handle unexpected HTTP 2XX responses

dimcli/core/api.py

@@ -14,6 +14,7 @@
 import IPython.display
 from itertools import islice
 import urllib.parse
+from requests.exceptions import HTTPError
 
 import pandas as pd
 
@@ -193,11 +194,18 @@ def query(self, q, show_results=None, retry=0, verbose=None):
                 if verbose: printDebug("ERROR LOG\n---\nQuery\n---\n" + str(q), "red")
                 if verbose: printDebug("Response.header\n---\n" + str(response.headers), "red")
                 if verbose: printDebug("Response.content\n---\n" +str(response.content), "red")
+
                 response.raise_for_status()
 
+                # raise_for_status() doesn't treat 2XX messages as an error but empty HTTP 202 
+                # messages have been observed.
+                #
+                # Any HTTP response that has made it here needs to trigger an exception or else we 
+                # will return None, which can cause problem for calling code.
 
+                raise HTTPError(f"Unexpected response HTTP {response.status_code}", response=response)
 
-    def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, force=False, maxlimit=0, verbose=None, _tot_count_prev_query=0, _warnings_tot=None):       
+    def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, force=False, maxlimit=0, verbose=None, retry=0, _tot_count_prev_query=0, _warnings_tot=None):       
         """Runs a DSL query and then keep querying until all matching records have been extracted. 
 
         The API returns a maximum of 1000 records per call. If a DSL query results in more than 1000 matches, it is possible to use pagination to get more results, up to 50k. 
@@ -224,7 +232,7 @@ def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, f
             The maximum number of records to extract in total. If 0, all available records are extracted, up to the API upper limit of 50k records per query.
         verbose : bool, default=False
             Verbose mode.
-
+        retry: number of retries per individual request, when an error is encountered
 
         Returns
         -------
@@ -291,7 +299,7 @@ def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, f
         q2 = q + " limit %d skip %d" % (limit, skip)
 
         start = time.time()
-        res = self.query(q2, show_results=False, retry=0, verbose=False)
+        res = self.query(q2, show_results=False, retry=retry, verbose=False)
         end = time.time()
         elapsed = end - start
 
@@ -314,7 +322,7 @@ def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, f
             tot =  _tot_count_prev_query # when force=True, we have no current query stats
 
         new_skip = skip+limit
-        if tot > 0 and new_skip > tot:
+        if tot is not None and tot > 0 and new_skip > tot:
             new_skip = tot
         if verbose and tot:  # if not first iteration
             t = "%.2f" % elapsed
@@ -1011,4 +1019,4 @@ def __repr__(self):
 # 2019-12-17: for backward compatibility
 # remove once all notebooks code has been updated 
 Result = DslDataset
-Dataset = DslDataset
+Dataset = DslDataset