From 1ab731679c8ff917662b8952a3d75e3300a01bd5 Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 19:56:41 +0100 Subject: [PATCH 1/9] Policy for Cloudwatch --- .github/workflows/ci_cd.yml | 24 ++++++++++++------------ extract_api_data/api_data.py | 1 - terraform/main.tf | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index e20f7b7..ce7fda3 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -28,18 +28,18 @@ jobs: run: make down - # deploy_terraform: - # runs-on: ubuntu-latest - # needs: [ci_job] - # steps: - # - name: checkout repo - # uses: actions/checkout@v2 - - # - name: init terraform - # run: make infra-init - - # - name: apply terraform - # run: make infra-apply + deploy_terraform: + runs-on: ubuntu-latest + needs: [ci_job] + steps: + - name: checkout repo + uses: actions/checkout@v2 + + - name: init terraform + run: make infra-init + + - name: apply terraform + run: make infra-apply push_docker_image: diff --git a/extract_api_data/api_data.py b/extract_api_data/api_data.py index 8c36ef1..481dc33 100644 --- a/extract_api_data/api_data.py +++ b/extract_api_data/api_data.py @@ -49,5 +49,4 @@ def lambda_handler(event, context): key = f"api_data/{today_date}/movies.json" data = extract_api_data(url, headers) - json_data = json.dumps(data) upload_to_s3(bucket_name, key, data) diff --git a/terraform/main.tf b/terraform/main.tf index 9c22d13..abf4374 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -50,5 +50,5 @@ resource "aws_iam_role" "lambda_iam_role" { ] }) - managed_policy_arns = ["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess", "arn:aws:iam::aws:policy/AmazonS3FullAccess","arn:aws:iam::aws:policy/CloudWatchEventsFullAccess"] + managed_policy_arns = ["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess", "arn:aws:iam::aws:policy/AmazonS3FullAccess","arn:aws:iam::aws:policy/CloudWatchEventsFullAccess","arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"] } From 01bb7d4488b838770100467640349ecf5c96132f Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 19:57:18 +0100 Subject: [PATCH 2/9] edit cicd --- .github/workflows/ci_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index ce7fda3..3e352b0 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -44,7 +44,7 @@ jobs: push_docker_image: runs-on: ubuntu-latest - needs: [ci_job] + needs: [ci_job,deploy_terraform] steps: - name: checkout repo uses: actions/checkout@v2 From aa1e3636f79fe12d95cd2ce45d11d1e8cc7a399b Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 22:24:05 +0100 Subject: [PATCH 3/9] edit lambda code-requests --- extract_api_data/api_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extract_api_data/api_data.py b/extract_api_data/api_data.py index 481dc33..b1f9f88 100644 --- a/extract_api_data/api_data.py +++ b/extract_api_data/api_data.py @@ -17,7 +17,7 @@ def extract_api_data(url: str, headers: dict) -> dict: try: response = requests.get(url, headers=headers) if response.status_code == 200: - data = json.loads(response.text)['results'] + data = response.json()['results'] logger.info('API data extracted') return data except Exception as e: @@ -41,6 +41,7 @@ def lambda_handler(event, context): 'https://api.themoviedb.org/3/discover/movie?include_adult=' + 'false&include_video=false&language=en-US&page=1&sort_by=popularity.desc' ) + headers = {"accept": "application/json", "Authorization": authorization} today_date = datetime.date.today().strftime("%Y-%m-%d") From a91b08af6d6b3bf35076b44a06a78cc6878c89d1 Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 22:31:45 +0100 Subject: [PATCH 4/9] rmv env variable --- extract_api_data/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_api_data/Dockerfile b/extract_api_data/Dockerfile index db5747f..23061c1 100644 --- a/extract_api_data/Dockerfile +++ b/extract_api_data/Dockerfile @@ -1,6 +1,6 @@ FROM public.ecr.aws/lambda/python:3.12 -ENV Authorization=${AUTHORIZATION} +ENV Authorization="Bearer eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhMTg3NWNjOTlkOGZhY2ZkZTg2OTJkMjk4MTQ5NjQ1YiIsInN1YiI6IjY1OTk5OGU4YmQ1ODhiMDFhOTU3M2FjNyIsInNjb3BlcyI6WyJhcGlfcmVhZCJdLCJ2ZXJzaW9uIjoxfQ.Z0lbJN4SXfdHU9dJRlHGr7hO0HtvZagtMFD7l4L1W6E" WORKDIR ${LAMBDA_TASK_ROOT} From 0b93b311642da62c5f5a1e569fa25feecaae9d9e Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 22:47:01 +0100 Subject: [PATCH 5/9] add env var --- extract_api_data/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_api_data/Dockerfile b/extract_api_data/Dockerfile index 23061c1..db5747f 100644 --- a/extract_api_data/Dockerfile +++ b/extract_api_data/Dockerfile @@ -1,6 +1,6 @@ FROM public.ecr.aws/lambda/python:3.12 -ENV Authorization="Bearer eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhMTg3NWNjOTlkOGZhY2ZkZTg2OTJkMjk4MTQ5NjQ1YiIsInN1YiI6IjY1OTk5OGU4YmQ1ODhiMDFhOTU3M2FjNyIsInNjb3BlcyI6WyJhcGlfcmVhZCJdLCJ2ZXJzaW9uIjoxfQ.Z0lbJN4SXfdHU9dJRlHGr7hO0HtvZagtMFD7l4L1W6E" +ENV Authorization=${AUTHORIZATION} WORKDIR ${LAMBDA_TASK_ROOT} From e20f8f0018c8a4cd32748ae3a65670fffa0e555f Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 22:50:51 +0100 Subject: [PATCH 6/9] env --- .github/workflows/ci_cd.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 3e352b0..8d81d27 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -28,23 +28,23 @@ jobs: run: make down - deploy_terraform: - runs-on: ubuntu-latest - needs: [ci_job] - steps: - - name: checkout repo - uses: actions/checkout@v2 + # deploy_terraform: + # runs-on: ubuntu-latest + # needs: [ci_job] + # steps: + # - name: checkout repo + # uses: actions/checkout@v2 - - name: init terraform - run: make infra-init + # - name: init terraform + # run: make infra-init - - name: apply terraform - run: make infra-apply + # - name: apply terraform + # run: make infra-apply push_docker_image: runs-on: ubuntu-latest - needs: [ci_job,deploy_terraform] + needs: [ci_job] steps: - name: checkout repo uses: actions/checkout@v2 @@ -67,7 +67,7 @@ jobs: IMAGE_TAG: ${{ github.sha }} run: | cd extract_api_data - docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . + docker build --build-arg AUTHORIZATION=${AUTHORIZATION} -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG aws lambda update-function-code \ --function-name test-lambda \ From fb8425addcd929fdfc3d856102b18d179b4a666e Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 22:55:56 +0100 Subject: [PATCH 7/9] args docker --- .github/workflows/ci_cd.yml | 2 +- extract_api_data/Dockerfile | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 8d81d27..3132414 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -67,7 +67,7 @@ jobs: IMAGE_TAG: ${{ github.sha }} run: | cd extract_api_data - docker build --build-arg AUTHORIZATION=${AUTHORIZATION} -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . + docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . --build-arg AUTHORIZATION=${AUTHORIZATION} docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG aws lambda update-function-code \ --function-name test-lambda \ diff --git a/extract_api_data/Dockerfile b/extract_api_data/Dockerfile index db5747f..923bd1f 100644 --- a/extract_api_data/Dockerfile +++ b/extract_api_data/Dockerfile @@ -1,6 +1,8 @@ FROM public.ecr.aws/lambda/python:3.12 -ENV Authorization=${AUTHORIZATION} +ARG AUTHORIZATION + +ENV Authorization=AUTHORIZATION WORKDIR ${LAMBDA_TASK_ROOT} From 0f917eb81a501568b07750d6f8b9ff7826e293d5 Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 23:00:33 +0100 Subject: [PATCH 8/9] docker args --- .github/workflows/ci_cd.yml | 2 +- extract_api_data/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 3132414..20b16b5 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -67,7 +67,7 @@ jobs: IMAGE_TAG: ${{ github.sha }} run: | cd extract_api_data - docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . --build-arg AUTHORIZATION=${AUTHORIZATION} + docker buildx build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . --build-arg AUTHORIZATION=${AUTHORIZATION} docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG aws lambda update-function-code \ --function-name test-lambda \ diff --git a/extract_api_data/Dockerfile b/extract_api_data/Dockerfile index 923bd1f..1caef8a 100644 --- a/extract_api_data/Dockerfile +++ b/extract_api_data/Dockerfile @@ -2,7 +2,7 @@ FROM public.ecr.aws/lambda/python:3.12 ARG AUTHORIZATION -ENV Authorization=AUTHORIZATION +ENV Authorization=${AUTHORIZATION} WORKDIR ${LAMBDA_TASK_ROOT} From 9af854d8900e4c3a22c492c3658a9ec3f8ac6660 Mon Sep 17 00:00:00 2001 From: Dorian Date: Thu, 8 Feb 2024 23:05:18 +0100 Subject: [PATCH 9/9] delete auth args --- .github/workflows/ci_cd.yml | 3 +-- extract_api_data/Dockerfile | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 20b16b5..f76763f 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -8,7 +8,6 @@ on: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AUTHORIZATION: ${{ secrets.AUTHORIZATION }} jobs: @@ -67,7 +66,7 @@ jobs: IMAGE_TAG: ${{ github.sha }} run: | cd extract_api_data - docker buildx build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . --build-arg AUTHORIZATION=${AUTHORIZATION} + docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG aws lambda update-function-code \ --function-name test-lambda \ diff --git a/extract_api_data/Dockerfile b/extract_api_data/Dockerfile index 1caef8a..1a2a194 100644 --- a/extract_api_data/Dockerfile +++ b/extract_api_data/Dockerfile @@ -1,9 +1,6 @@ FROM public.ecr.aws/lambda/python:3.12 -ARG AUTHORIZATION - -ENV Authorization=${AUTHORIZATION} - +ENV Authorization="Bearer eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJhMTg3NWNjOTlkOGZhY2ZkZTg2OTJkMjk4MTQ5NjQ1YiIsInN1YiI6IjY1OTk5OGU4YmQ1ODhiMDFhOTU3M2FjNyIsInNjb3BlcyI6WyJhcGlfcmVhZCJdLCJ2ZXJzaW9uIjoxfQ.Z0lbJN4SXfdHU9dJRlHGr7hO0HtvZagtMFD7l4L1W6E" WORKDIR ${LAMBDA_TASK_ROOT} COPY requirements.txt ./