Skip to content

feat(parameters): add get_parameters_by_name for SSM params in distinct paths #1678

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
fff76db
chore(deps): add typing_extensions dep
heitorlessa Nov 1, 2022
d238286
feat(parameters): initial prototype for get_parameters_by_name
heitorlessa Nov 1, 2022
a22f7db
feat(parameters): add multi-thread option
heitorlessa Nov 2, 2022
e13bbef
docs(parameters): add get_parameters_by_name
heitorlessa Nov 2, 2022
9dc4ce6
chore(tests): add end-to-end test
heitorlessa Nov 2, 2022
1c6c71c
chore(parameters): remove parallel option due to timeout risk
heitorlessa Nov 2, 2022
215b2d7
refactor: strict typing transform_value/method
heitorlessa Nov 3, 2022
56df26f
refactor: move to GetParameters, use GetParameter upon decrypt
heitorlessa Nov 3, 2022
843ee9e
fix(parameters): transform_value auto should work for both single and…
heitorlessa Nov 4, 2022
0a294f5
chore(tests): add functional test for decrypt, batch split, and overr…
heitorlessa Nov 4, 2022
dd3b2b3
chore(tests): add functional test for cache batch, transform override
heitorlessa Nov 4, 2022
0faad69
feat(parameters): graceful error handling for raise_on_failure; cleanup
heitorlessa Nov 6, 2022
8e3a7a9
feat(parameters): expose has_not_expired_in_cache method to ease tests
heitorlessa Nov 7, 2022
4d63a22
chore(tests): ensure null or negative max_age params aren't cached
heitorlessa Nov 7, 2022
6613a23
refactor: break logic in multiple methods to ease maintenance
heitorlessa Nov 7, 2022
f213dea
chore: add docstring with example
heitorlessa Nov 7, 2022
859171b
docs(parameters): document graceful error handling for get_parameters…
heitorlessa Nov 7, 2022
e0a63ae
feat(parameters): add guardrail for param also named _errors in grace…
heitorlessa Nov 7, 2022
386e813
docs: add IAM permission for get_parameters_by_name
heitorlessa Nov 7, 2022
27d05c4
feat: use GetParameters if entire batch needs decryption
heitorlessa Nov 7, 2022
aa5670c
docs: add ascii diagram to ease understanding of batch split API
heitorlessa Nov 7, 2022
3d763bc
Merge branch 'develop' into feat/get-parameters-by-name
heitorlessa Nov 7, 2022
fbffe48
chore: ignore assignment mypy due to dinamism of transform
heitorlessa Nov 7, 2022
d205ca4
chore(tests): remove redundant override test
heitorlessa Nov 7, 2022
b952152
fix(tests): boto3 client side effect on super class
heitorlessa Nov 7, 2022
9eaf0f4
fix(tests): compat remove_prefix for 3.7+
heitorlessa Nov 7, 2022
9fc46a7
chore(mypy): ignore assignment overload value
heitorlessa Nov 7, 2022
285c431
chore(mypy): ignore call-overload and assignment overload value
heitorlessa Nov 7, 2022
9e84a07
fix(tests): boto3 client side effect on super class
heitorlessa Nov 7, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
chore(parameters): remove parallel option due to timeout risk
  • Loading branch information
heitorlessa committed Nov 2, 2022
commit 1c6c71c170d06ca35307be5af00c196f63eae499
74 changes: 17 additions & 57 deletions aws_lambda_powertools/utilities/parameters/ssm.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
"""
AWS SSM Parameter retrieval and caching utility
"""
import concurrent.futures
import functools
from concurrent.futures import Future
from typing import TYPE_CHECKING, Any, Dict, Optional, Union, overload

import boto3
Expand Down Expand Up @@ -356,7 +353,6 @@ def get_parameters_by_name(
decrypt: bool = False,
force_fetch: bool = False,
max_age: int = DEFAULT_MAX_AGE_SECS,
parallel: bool = False,
) -> Dict[str, str]:
...

Expand All @@ -368,7 +364,6 @@ def get_parameters_by_name(
decrypt: bool = False,
force_fetch: bool = False,
max_age: int = DEFAULT_MAX_AGE_SECS,
parallel: bool = False,
) -> Dict[str, bytes]:
...

Expand All @@ -380,7 +375,6 @@ def get_parameters_by_name(
decrypt: bool = False,
force_fetch: bool = False,
max_age: int = DEFAULT_MAX_AGE_SECS,
parallel: bool = False,
) -> Dict[str, Dict[str, Any]]:
...

Expand All @@ -392,7 +386,6 @@ def get_parameters_by_name(
decrypt: bool = False,
force_fetch: bool = False,
max_age: int = DEFAULT_MAX_AGE_SECS,
parallel: bool = False,
) -> Union[Dict[str, str], Dict[str, dict]]:
...

Expand All @@ -403,7 +396,6 @@ def get_parameters_by_name(
decrypt: bool = False,
force_fetch: bool = False,
max_age: int = DEFAULT_MAX_AGE_SECS,
parallel: bool = False,
) -> Union[Dict[str, str], Dict[str, bytes], Dict[str, dict]]:
"""
Retrieve multiple parameter values by name from AWS Systems Manager (SSM) Parameter Store
Expand All @@ -420,8 +412,6 @@ def get_parameters_by_name(
Force update even before a cached item has expired, defaults to False
max_age: int
Maximum age of the cached value
sdk_options: dict, optional
Dictionary of options that will be passed to the Parameter Store get_parameter API call

Raises
------
Expand All @@ -432,54 +422,24 @@ def get_parameters_by_name(
When the parameter provider fails to transform a parameter value.
"""

# NOTE: Need a param for hard failure mode on parameter retrieval (asked feature request author)
# NOTE: Decide whether to leave multi-threaded option or not due to slower results (throttling+LWP cost)
# NOTE: Decided against using multi-thread due to single-thread outperforming in 128M and 1G + timeout risk
# see: https://github.com/awslabs/aws-lambda-powertools-python/issues/1040#issuecomment-1299954613

ret: Dict[str, Any] = {}
future_to_param: Dict[Future, str] = {}

if parallel:
with concurrent.futures.ThreadPoolExecutor(max_workers=len(parameters)) as pool:
for parameter, options in parameters.items():
if isinstance(options, dict):
transform = options.get("transform") or transform
decrypt = options.get("decrypt") or decrypt
max_age = options.get("max_age") or max_age
force_fetch = options.get("force_fetch") or force_fetch

fetch_parameter_callable = functools.partial(
get_parameter,
name=parameter,
transform=transform,
decrypt=decrypt,
max_age=max_age,
force_fetch=force_fetch,
)

future = pool.submit(fetch_parameter_callable)
future_to_param[future] = parameter

for future in concurrent.futures.as_completed(future_to_param):
try:
# "parameter": "future result"
ret[future_to_param[future]] = future.result()
except Exception as exc:
print(f"Uh oh, failed to fetch '{future_to_param[future]}': {exc}")

else:
for parameter, options in parameters.items():
if isinstance(options, dict):
transform = options.get("transform") or transform
decrypt = options.get("decrypt") or decrypt
max_age = options.get("max_age") or max_age
force_fetch = options.get("force_fetch") or force_fetch

ret[parameter] = get_parameter(
name=parameter,
transform=transform,
decrypt=decrypt,
max_age=max_age,
force_fetch=force_fetch,
)

for parameter, options in parameters.items():
if isinstance(options, dict):
transform = options.get("transform") or transform
decrypt = options.get("decrypt") or decrypt
max_age = options.get("max_age") or max_age
force_fetch = options.get("force_fetch") or force_fetch

ret[parameter] = get_parameter(
name=parameter,
transform=transform,
decrypt=decrypt,
max_age=max_age,
force_fetch=force_fetch,
)

return ret
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ def lambda_handler(event: dict, context: LambdaContext) -> Dict[str, Any]:
parameters_to_fetch: Dict[str, Any] = {param: {} for param in parameters_list}

# response`{parameter:value}`
return get_parameters_by_name(parameters=parameters_to_fetch, max_age=0, parallel=True)
return get_parameters_by_name(parameters=parameters_to_fetch, max_age=0)