data-engineering-with-python/rest-api.py at main · jeantardelli/data-engineering-with-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
"""
This module uses Python to query different endpoints using the
NiFi REST API.
"""
import requests
import pprint

# System Diagnostics
r = requests.get('http://localhost:9300/nifi-api/system-diagnostics')
data = r.json()

print('Aggregte Snapshot:\n'
      '==================')
print('Max Heap: {0}'.format(
    data['systemDiagnostics']['aggregateSnapshot']['maxHeap']))
print('Total Threads: {0}'.format(
    data['systemDiagnostics']['aggregateSnapshot']['totalThreads']))
print('Heap Utilization: {0}'.format(
    data['systemDiagnostics']['aggregateSnapshot']['heapUtilization']))
print('')

# Processor Groups
pg = requests.get('http://localhost:9300/nifi-api/process-groups/01761002-6e6d-1e28-3038-c32c45328032')
pgdata = pg.json()

print('Processor Group:\n'
      '================')
print('Component Name: {0}'.format(pgdata['component']['name']))
print('Status:')
pprint.pprint(pgdata['status'])
print('')

# Single Processor
p = requests.get('http://localhost:9300/nifi-api/processors/215ab91d-90c7-3e63-219e-80a21faee5c8')
pdata = p.json()

print('Single Processor:\n'
      '=================')
print('Component name: {0}'.format(pdata['component']['name']))
print('Status:')
pprint.pprint(pdata['status'])
print('')

# Make a listing request to queue
q = requests.post(
        'http://localhost:9300/nifi-api/flowfile-queues/dcec54be-3777-36a0-1d06-be5fc54a25c3/listing-requests')
qdata = q.json()

print('Queue List:\n'
      '===========')
print('List id: {0}'.format(qdata['listingRequest']['id']))
print('')

url = "http://localhost:9300/nifi-api/flowfile-queues/" + \
      "dcec54be-3777-36a0-1d06-be5fc54a25c3/listing-requests/" + \
      qdata['listingRequest']['id']

ff = requests.get(url)
ffdata = ff.json()

print('Request uuid:\n'
      '=============')
print('Id: {0}'.format(ffdata['listingRequest']['flowFileSummaries'][0]['uuid']))
print('')

ffurl = "http://localhost:9300/nifi-api/flowfile-queues/" + \
      "dcec54be-3777-36a0-1d06-be5fc54a25c3/flowfiles/" + \
      ffdata['listingRequest']['flowFileSummaries'][0]['uuid'] + \
      "/content"

print('Request Content:\n'
      '================')
print('Content:')
download = requests.get(ffurl)
pprint.pprint(download.json())
print('')

# Empty queue
e = requests.post('http://localhost:9300/nifi-api/flowfile-queues/' + \
        'dcec54be-3777-36a0-1d06-be5fc54a25c3/drop-requests')
edata = e.json()

# Read Bulletin
b = requests.get('http://localhost:9300/nifi-api/flow/bulletin-board')
bdata = b.json()
print('Bulletin:\n'
      '=========')
print('Content:')
pprint.pprint(bdata)
print('')

# Read Counters
c = requests.get('http://localhost:9300/nifi-api/counters')
cdata = c.json()
print('Counters:\n'
      '=========')
pprint.pprint(cdata)