|
|
|
@ -83,7 +83,9 @@ def fetch_daily_maximo_data(**context):
|
|
|
|
except requests.exceptions.Timeout:
|
|
|
|
except requests.exceptions.Timeout:
|
|
|
|
logger.error(f"Request connection timed out (Request ID: {request_id})")
|
|
|
|
logger.error(f"Request connection timed out (Request ID: {request_id})")
|
|
|
|
except Exception as e:
|
|
|
|
except Exception as e:
|
|
|
|
logger.error(f"Error initiating request (Request ID: {request_id}): {str(e)}")
|
|
|
|
logger.error(
|
|
|
|
|
|
|
|
f"Error initiating request (Request ID: {request_id}): {str(e)}"
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
# Using ThreadPoolExecutor for async operation
|
|
|
|
# Using ThreadPoolExecutor for async operation
|
|
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
|
|
@ -146,7 +148,7 @@ default_args = {
|
|
|
|
|
|
|
|
|
|
|
|
# Define the DAG
|
|
|
|
# Define the DAG
|
|
|
|
dag = DAG(
|
|
|
|
dag = DAG(
|
|
|
|
"fetch_daily_maximo_data_async",
|
|
|
|
"fetch_daily_maximo_data",
|
|
|
|
default_args=default_args,
|
|
|
|
default_args=default_args,
|
|
|
|
description="A DAG to fetch data from Maximo API endpoint asynchronously on a daily schedule",
|
|
|
|
description="A DAG to fetch data from Maximo API endpoint asynchronously on a daily schedule",
|
|
|
|
# Schedule to run daily at 21:00, 22:00, and 23:00
|
|
|
|
# Schedule to run daily at 21:00, 22:00, and 23:00
|
|
|
|
|