-
Notifications
You must be signed in to change notification settings - Fork 26
/
Copy pathinstances.py
515 lines (419 loc) · 17.7 KB
/
instances.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
from .future import FMFuture
from ..dssclient import DSSClient
from ..govern_client import GovernClient
import sys
if sys.version_info > (3, 4):
from enum import Enum
else:
class Enum(object):
pass
class FMInstanceCreator(object):
def __init__(
self, client, label, instance_settings_template_id, virtual_network_id, image_id
):
"""
Helper to create a DSS instance.
:param client: The FM client
:type client: :class:`dataikuapi.fm.fmclient`
:param str label: The label of the instance
:param str instance_settings_template: The instance settings template id this instance should be based on
:param str virtual_network: The virtual network where the instance should be spawned
:param str image_id: The ID of the DSS runtime image (ex: dss-9.0.3-default)
"""
self.client = client
self.data = {}
self.data["label"] = label
self.data["instanceSettingsTemplateId"] = instance_settings_template_id
self.data["virtualNetworkId"] = virtual_network_id
self.data["imageId"] = image_id
# Set the default value for dssNodeType
self.data["dssNodeType"] = "design"
def with_dss_node_type(self, dss_node_type):
"""
Set the DSS node type of the instance to create. The default node type is `DESIGN`.
:param dss_node_type: the type of the dss node to create.
:type dss_node_type: :class:`dataikuapi.fm.instances.FMNodeType`
:rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator`
"""
# backward compatibility, was a string before . be sure the value falls into the enum
value = dss_node_type
# python2 does not support enum, keep string
if sys.version_info > (3, 4):
if isinstance(dss_node_type, str):
value = FMNodeType[dss_node_type.upper()]
self.data["dssNodeType"] = value.value
else:
value = FMNodeType.get_from_string(dss_node_type.upper())
self.data["dssNodeType"] = value;
return self
def with_cloud_instance_type(self, cloud_instance_type):
"""
Set the machine type for the DSS Instance
:param str cloud_instance_type: the machine type to be used for the instance
:rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator`
"""
self.data["cloudInstanceType"] = cloud_instance_type
return self
def with_data_volume_options(
self,
data_volume_type=None,
data_volume_size=None,
data_volume_size_max=None,
data_volume_IOPS=None,
data_volume_encryption=None,
data_volume_encryption_key=None,
):
"""
Set the options of the data volume to use with the DSS Instance
:param str data_volume_type: Optional, data volume type
:param int data_volume_size: Optional, data volume initial size
:param int data_volume_size_max: Optional, data volume maximum size
:param int data_volume_IOPS: Optional, data volume IOPS
:param data_volume_encryption: Optional, encryption mode of the data volume
:type data_volume_encryption: :class:`dataikuapi.fm.instances.FMInstanceEncryptionMode`
:param str data_volume_encryption_key: Optional, the encryption key to use when data_volume_encryption_key is FMInstanceEncryptionMode.CUSTOM
:rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator`
"""
if not data_volume_encryption:
data_volume_encryption = FMInstanceEncryptionMode.NONE
if type(data_volume_encryption) is not FMInstanceEncryptionMode:
raise TypeError(
"data_volume encryption needs to be of type FMInstanceEncryptionMode"
)
self.data["dataVolumeType"] = data_volume_type
self.data["dataVolumeSizeGB"] = data_volume_size
self.data["dataVolumeSizeMaxGB"] = data_volume_size_max
self.data["dataVolumeIOPS"] = data_volume_IOPS
self.data["volumesEncryption"] = data_volume_encryption.value
self.data["volumesEncryptionKey"] = data_volume_encryption_key
return self
def with_cloud_tags(self, cloud_tags):
"""
Set the tags to be applied to the cloud resources created for this DSS instance
:param dict cloud_tags: a key value dictionary of tags to be applied on the cloud resources
:rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator`
"""
self.data["cloudTags"] = cloud_tags
return self
def with_fm_tags(self, fm_tags):
"""
A list of tags to add on the DSS Instance in Fleet Manager
:param list fm_tags: Optional, list of tags to be applied on the instance in the Fleet Manager
:rtype: :class:`dataikuapi.fm.instances.FMInstanceCreator`
"""
self.data["fmTags"] = fm_tags
return self
class FMAWSInstanceCreator(FMInstanceCreator):
def with_aws_root_volume_options(
self,
aws_root_volume_size=None,
aws_root_volume_type=None,
aws_root_volume_IOPS=None,
):
"""
Set the options of the root volume of the DSS Instance
:param int aws_root_volume_size: Optional, the root volume size
:param str aws_root_volume_type: Optional, the root volume type
:param int aws_root_volume_IOPS: Optional, the root volume IOPS
:rtype: :class:`dataikuapi.fm.instances.FMAWSInstanceCreator`
"""
self.data["awsRootVolumeSizeGB"] = aws_root_volume_size
self.data["awsRootVolumeType"] = aws_root_volume_type
self.data["awsRootVolumeIOPS"] = aws_root_volume_IOPS
return self
def create(self):
"""
Create the DSS instance
:return: a newly created DSS instance
:rtype: :class:`dataikuapi.fm.instances.FMAWSInstance`
"""
instance = self.client._perform_tenant_json(
"POST", "/instances", body=self.data
)
return FMAWSInstance(self.client, instance)
class FMAzureInstanceCreator(FMInstanceCreator):
def create(self):
"""
Create the DSS instance
:return: a newly created DSS instance
:rtype: :class:`dataikuapi.fm.instances.FMAzureInstance`
"""
instance = self.client._perform_tenant_json(
"POST", "/instances", body=self.data
)
return FMAzureInstance(self.client, instance)
class FMGCPInstanceCreator(FMInstanceCreator):
def create(self):
"""
Create the DSS instance
:return: a newly created DSS instance
:rtype: :class:`dataikuapi.fm.instances.FMGCPInstance`
"""
instance = self.client._perform_tenant_json(
"POST", "/instances", body=self.data
)
return FMGCPInstance(self.client, instance)
class FMInstance(object):
"""
A handle to interact with a DSS instance.
Do not create this directly, use :meth:`dataikuapi.fmclient.FMClient.get_instance` or
* :meth:`dataikuapi.fmclient.FMClientAWS.new_instance_creator`
* :meth:`dataikuapi.fmclient.FMClientAzure.new_instance_creator`
* :meth:`dataikuapi.fmclient.FMClientGCP.new_instance_creator`
"""
def __init__(self, client, instance_data):
self.client = client
self.instance_data = instance_data
self.id = instance_data["id"]
def get_client(self):
"""
Get a Python client to communicate with a DSS instance
:return: a Python client to communicate with the target instance
:rtype: :class:`dataikuapi.dssclient.DSSClient`
"""
instance_status = self.get_status()
external_url = instance_status.get("publicURL")
admin_api_key_resp = self.client._perform_tenant_json(
"GET", "/instances/%s/admin-api-key" % self.id
)
api_key = admin_api_key_resp["adminAPIKey"]
if not external_url:
raise ValueError("No external URL available for node %s. This node may not be provisioned yet" % self.id)
if self.instance_data.get("dssNodeType") == "govern":
return GovernClient(external_url, api_key)
else:
return DSSClient(external_url, api_key)
def reprovision(self):
"""
Reprovision the physical DSS instance
:return: the `Future` object representing the reprovision process
:rtype: :class:`dataikuapi.fm.future.FMFuture`
"""
future = self.client._perform_tenant_json(
"GET", "/instances/%s/actions/reprovision" % self.id
)
return FMFuture.from_resp(self.client, future)
def deprovision(self):
"""
Deprovision the physical DSS instance
:return: the `Future` object representing the deprovision process
:rtype: :class:`dataikuapi.fm.future.FMFuture`
"""
future = self.client._perform_tenant_json(
"GET", "/instances/%s/actions/deprovision" % self.id
)
return FMFuture.from_resp(self.client, future)
def restart_dss(self):
"""
Restart the DSS running on the physical instance
:return: the `Future` object representing the restart process
:rtype: :class:`dataikuapi.fm.future.FMFuture`
"""
future = self.client._perform_tenant_json(
"GET", "/instances/%s/actions/restart-dss" % self.id
)
return FMFuture.from_resp(self.client, future)
def save(self):
"""
Update the instance
"""
self.client._perform_tenant_empty(
"PUT", "/instances/%s" % self.id, body=self.instance_data
)
self.instance_data = self.client._perform_tenant_json(
"GET", "/instances/%s" % self.id
)
def get_status(self):
"""
Get the physical DSS instance's status
:return: the instance status
:rtype: :class:`dataikuapi.fm.instances.FMInstanceStatus`
"""
status = self.client._perform_tenant_json(
"GET", "/instances/%s/status" % self.id
)
return FMInstanceStatus(status)
def delete(self):
"""
Delete the DSS instance
:return: the `Future` object representing the deletion process
:rtype: :class:`dataikuapi.fm.future.FMFuture`
"""
future = self.client._perform_tenant_json(
"GET", "/instances/%s/actions/delete" % self.id
)
return FMFuture.from_resp(self.client, future)
def get_initial_password(self):
"""
Get the initial DSS admin password.
Can only be called once
:return: a password for the 'admin' user.
"""
return self.client._perform_tenant_json(
"GET", "/instances/%s/actions/get-initial-password" % self.id
)
def reset_user_password(self, username, password):
"""
Reset the password for a user on the DSS instance
:param string username: login
:param string password: new password
:return: the `Future` object representing the password reset process
:rtype: :class:`dataikuapi.fm.future.FMFuture`
"""
future = self.client._perform_tenant_json(
"GET", "/instances/%s/actions/reset-user-password" % self.id, params={ 'userName':username, 'password':password }
)
return FMFuture.from_resp(self.client, future)
def replay_setup_actions(self):
"""
Replay the setup actions on the DSS instance
:return: the `Future` object representing the replay process
:rtype: :class:`dataikuapi.fm.future.FMFuture`
"""
future = self.client._perform_tenant_json(
"GET", "/instances/%s/actions/replay-setup-actions" % self.id
)
return FMFuture.from_resp(self.client, future)
def set_automated_snapshots(self, enable, period, keep=0):
"""
Set the automated snapshot policy for this instance
:param boolean enable: Enable the automated snapshots
:param int period: The time period between 2 snapshot in hours
:param int keep: Optional, the number of snapshot to keep. Use 0 to keep all snapshots. Defaults to 0.
"""
self.instance_data["enableAutomatedSnapshot"] = enable
self.instance_data["automatedSnapshotPeriod"] = period
self.instance_data["automatedSnapshotRetention"] = keep
return self
def set_custom_certificate(self, pem_data):
"""
Set the custom certificate for this instance
Only needed when Virtual Network HTTPS Strategy is set to Custom Certificate
:param str pem_data: The SSL certificate
"""
self.instance_data["sslCertificatePEM"] = pem_data
return self
########################################################
# Snapshots
########################################################
def list_snapshots(self):
"""
List all the snapshots of this instance
:return: list of snapshots
:rtype: list of :class:`dataikuapi.fm.instances.FMSnapshot`
"""
snapshots = self.client._perform_tenant_json("GET", "/instances/%s/snapshots" % self.id)
return [FMSnapshot(self.client, self.id, x["id"], x) for x in snapshots]
def get_snapshot(self, snapshot_id):
"""
Get a snapshot of this instance
:param str snapshot_id: identifier of the snapshot
:return: Snapshot
:rtype: :class:`dataikuapi.fm.instances.FMSnapshot`
"""
return FMSnapshot(self.client, self.id, snapshot_id)
def snapshot(self, reason_for_snapshot=None):
"""
Create a snapshot of the instance
:return: Snapshot
:rtype: :class:`dataikuapi.fm.instances.FMSnapshot`
"""
snapshot = self.client._perform_tenant_json(
"POST", "/instances/%s/snapshots" % self.id, params={ "reasonForSnapshot":reason_for_snapshot }
)
return FMSnapshot(self.client, self.id, snapshot["id"], snapshot)
class FMAWSInstance(FMInstance):
def set_elastic_ip(self, enable, elasticip_allocation_id):
"""
Set a public elastic ip for this instance
:param boolan enable: Enable the elastic ip allocation
:param str elasticip_allocation_id: AWS ElasticIP allocation ID
"""
self.instance_data["awsAssignElasticIP"] = enable
self.instance_data["awsElasticIPAllocationId"] = elasticip_allocation_id
return self
class FMAzureInstance(FMInstance):
def set_elastic_ip(self, enable, public_ip_id):
"""
Set a public elastic ip for this instance
:param boolan enable: Enable the elastic ip allocation
:param str public_ip_id: Azure Public IP ID
"""
self.instance_data["azureAssignElasticIP"] = enable
self.instance_data["azurePublicIPId"] = public_ip_id
return self
class FMGCPInstance(FMInstance):
def set_public_ip(self, enable, public_ip_id):
"""
Set a public ip for this instance
:param boolan enable: Enable the public ip allocation
:param str public_ip_id: GCP Public IP ID
"""
self.instance_data["gcpAssignPublicIP"] = enable
self.instance_data["gcpPublicIPId"] = public_ip_id
return self
class FMNodeType(Enum):
DESIGN = "design"
DEPLOYER = "deployer"
AUTOMATION = "automation"
GOVERN = "govern"
# Python2 emulated enum. to be removed on Python2 support removal
@staticmethod
def get_from_string(s):
if s == "DESIGN": return FMNodeType.DESIGN
if s == "DEPLOYER": return FMNodeType.DEPLOYER
if s == "AUTOMATION": return FMNodeType.AUTOMATION
if s == "GOVERN": return FMNodeType.GOVERN
raise Exception("Invalid Node Type " + s)
class FMInstanceEncryptionMode(Enum):
NONE = "NONE"
DEFAULT_KEY = "DEFAULT_KEY"
TENANT = "TENANT"
CUSTOM = "CUSTOM"
class FMInstanceStatus(dict):
"""A class holding read-only information about an Instance.
This class should not be created directly. Instead, use :meth:`FMInstance.get_status`
"""
def __init__(self, data):
"""Do not call this directly, use :meth:`FMInstance.get_status`"""
super(FMInstanceStatus, self).__init__(data)
class FMSnapshot(object):
"""
A handle to interact with a snapshot of a DSS instance.
Do not create this directly, use :meth:`FMInstance.snapshot`
"""
def __init__(self, client, instance_id, snapshot_id, snapshot_data=None):
self.client = client
self.instance_id = instance_id
self.snapshot_id = snapshot_id
self.snapshot_data = snapshot_data
def get_info(self):
"""
Get the information about this snapshot
:return: a dict
"""
if self.snapshot_data is None:
self.snapshot_data = self.client._perform_tenant_json(
"GET", "/instances/%s/snapshots/%s" % (self.instance_id, self.snapshot_id)
)
return self.snapshot_data
def reprovision(self):
"""
Reprovision the physical DSS instance from this snapshot
:return: the `Future` object representing the reprovision process
:rtype: :class:`dataikuapi.fm.future.FMFuture`
"""
future = self.client._perform_tenant_json(
"POST", "/instances/%s/snapshots/%s/reprovision" % (self.instance_id, self.snapshot_id)
)
return FMFuture.from_resp(self.client, future)
def delete(self):
"""
Delete the snapshot
:return: the `Future` object representing the deletion process
:rtype: :class:`dataikuapi.fm.future.FMFuture`
"""
future = self.client._perform_tenant_json(
"DELETE", "/instances/%s/snapshots/%s" % (self.instance_id, self.snapshot_id)
)
return FMFuture.from_resp(self.client, future)