diff --git a/.gitignore b/.gitignore index 1d74e21..416ffb6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .vscode/ +build_output/python +build_output/lib diff --git a/Dockerfile b/Dockerfile index 44603b5..26a07bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,19 @@ FROM lambci/lambda:build-python3.8 -ARG mysql_gpg_key_url="https://repo.mysql.com/RPM-GPG-KEY-mysql" -ARG mysql_gpg_key_name="RPM-GPG-KEY-mysql" -ARG mysql_repo_rpm="mysql80-community-release-el7-3.noarch.rpm" -ARG mysql_devel_package_url="https://dev.mysql.com/get/${mysql_repo_rpm}" -ARG mysql_devel_package="mysql-community-devel" -ARG python_package_to_install="mysqlclient" +# ARG mysql_gpg_key_url="https://repo.mysql.com/RPM-GPG-KEY-mysql" +# ARG mysql_gpg_key_name="RPM-GPG-KEY-mysql" +# ARG mysql_repo_rpm="mysql80-community-release-el7-3.noarch.rpm" +# ARG mysql_devel_package_url="https://dev.mysql.com/get/${mysql_repo_rpm}" +# ARG mysql_devel_package="mysql-community-devel" +# ARG python_package_to_install="mysqlclient" -# grab and import the MySQL repo GPG key to install mysql-devel later -RUN curl -Ls -c cookieJar -O ${mysql_gpg_key_url} -RUN rpm --import ${mysql_gpg_key_name} +# # grab and import the MySQL repo GPG key to install mysql-devel later +# RUN curl -Ls -c cookieJar -O ${mysql_gpg_key_url} +# RUN rpm --import ${mysql_gpg_key_name} -# prerequisite for getting mysql-devel package -RUN curl -Ls -c cookieJar -O ${mysql_devel_package_url} -RUN yum install -y ${mysql_repo_rpm} +# # prerequisite for getting mysql-devel package +# RUN curl -Ls -c cookieJar -O ${mysql_devel_package_url} +# RUN yum install -y ${mysql_repo_rpm} -# install mysql-devel package -RUN yum install -y ${mysql_devel_package} +# # install mysql-devel package +# RUN yum install -y ${mysql_devel_package} diff --git a/README.md b/README.md index 64de5d5..1a378ba 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,36 @@ -# AWS layer for `mysqlclient` for Python 3.x +# AWS Lambda layer for `mysqlclient` (or any other Python package) + +This project provides: + +1. Ready-made AWS layer zips for the Python [mysqlclient](https://github.com/PyMySQL/mysqlclient-python) (aka MySQLdb) package: for MySQL 5.6 and MySQL 8.0 +2. An easy, docker-based solution for building your own AWS layer: for `mysqlclient` for ANY version of MySQL server and ANY version of Python too. +3. An easy, docker-based, completely generalized solution for building your own AWS layer for ANY Python package for ANY Python version. This is especially useful for importing and using Python packages with platform-specific dependencies (e.g. the package uses `.so` files via FFI) in AWS Lambda. These packages are usually non-trivial to use in AWS Lambda for reasons described below. Example packages that fit these criteria: pandas, numpy, cchardet. If you have this use-case, use the `general-purpose` branch. ## TLDR -- Are you using Python 3.8? -- Are you using MySQL v8.0.x? -- Are you having trouble importing/using [mysqlclient](https://github.com/PyMySQL/mysqlclient-python) in your AWS Lambda function? -- Are you in a hurry? +If you need a ready-made, tested AWS Layer for `mysqlclient`, just use `build_output/layer.zip` according to this table: + +| Python Version | MySQL Version | Branch to use | +|---|---|---| +| 3.x | MySQL v8.0.x | master | +| 3.x | MySQL v5.6.x | mysql-5.6 | +| 3.x | I want to build an AWS Lambda layer for a non-MySQL Python package| general-purpose | + +If your use-case is not reflected in the table above (for example, you need to target a different version of MySQL and/or a different version of Python) then you can build your own AWS layer with the tools provided in this repo. Read on for more instructions. -If you answered `yes` to all the above questions, simply upload `layer.zip` from the `build_output` dir as an AWS Layer to your AWS account. -If you answered `no` to any of the above questions, then read on to figure out if you need to build your own `mysqlclient` AWS layer with the tools provided in this repo. +### Short HowTo -For details on how to upload a zip file as a new layer to AWS Lambda, see the section below called **create a new AWS layer with `layer.zip`**. +- Switch to the appropriate branch as per the table above. +- Either use the already-provided `build_output/layer.zip` or build your own `layer.zip` as described below. +- Upload `layer.zip` as an AWS Layer to your AWS account. For details on this step, see the [create a new AWS layer with `layer.zip`](https://github.com/nonbeing/mysqlclient-python3-aws-lambda#create-a-new-aws-layer-with-layerzip) section. +- Configure your AWS Lambda Python function to use your newly-created layer. +- Profit! -After you create the required layer, you can configure your Lambda function to use this layer and simply `import MySQLdb` without error in your Lambda function. + +## `mysqlclient`: simple, sufficient test for your new AWS Lambda layer + +After you create the required layer in AWS Lambda, you can configure your Lambda function to use this layer and simply `import MySQLdb` (as an example) in your Lambda function. The import should work just fine, without and errors if you are using the right Lambda layer and you have configured your Lambda function to use your Lambda layer correctly. Here's a barebones example to test if you are able to import and use `mysqlclient` correctly: @@ -27,7 +44,7 @@ def lambda_handler(event, context): } ``` -If you get the success message and don't see an error like `ModuleNotFoundError: No module named 'MySQLdb'` or `ImportError: No module named _mysql`, then you're all set to use `mysqlclient` on AWS Lambda. +If you get the success message and don't see an error like `ModuleNotFoundError: No module named 'MySQLdb'` or `ImportError: No module named _mysql`, then you're all set to use `mysqlclient` on AWS Lambda - the Lambda layer is working just fine for you. ## Motivation @@ -86,23 +103,46 @@ For most use-cases, `mysqlclient` is the preferred choice of DB connector to MyS This project attempts to solve (or at least alleviate) this problem to a large extent by providing a relatively-straightforward path to building an AWS Layer for `mysqlclient` which can then be readily consumed in AWS Lambda Python functions. -## Usage +# Usage -### prerequisites +## Building your own Lambda layer.zip for ANY Python package -You need a `*nix` environment where you can run docker commands and bash scripts (such as WSL2 on Windows 10 Pro, which has been tested). Tested on Ubuntu 20.04 +Skip the following steps if you are using any of the provided, ready-to-use `layer.zip` files. Go directly to [create a new AWS layer with `layer.zip`](https://github.com/nonbeing/mysqlclient-python3-aws-lambda#create-a-new-aws-layer-with-layerzip) instead. -Ensure you have docker installed. You will need to pull a [`lambci` image from docker hub](https://hub.docker.com/r/lambci/lambda). This image will be used to build a local docker image using the `Dockerfile` and install the appropriate `mysql-devel` package. +If you need to build your own Lambda layer from scratch, read on... + +### build prerequisites + +You need a `*nix` environment where you can run docker commands and bash scripts. This project has been tested on Ubuntu 20.04, WSL2 on Windows 10 Pro and MacOS. + +Ensure you have docker installed; you should be able to run `docker --version` without any issues. The script will pull a [`lambci` image from docker hub](https://hub.docker.com/r/lambci/lambda) as the first step of the build. This `lambci` image will be used to build a local docker image using the `Dockerfile`. If you are using one of the `mysql` branches or `master` branch, the script will also install the appropriate `mysql-devel` package which is necessary for building the `libmysqlclient.so`. ### build the layer locally -Simply clone this repo and invoke the `build.sh` shell script; it will perform all the steps required. +- Clone this repo. +- Switch to the appropriate branch (see table above). If you are not building for `mysqlclient`, use the `general-purpose` branch. +- Review the Dockerfile. You may need to edit this as per your needs (e.g. not building a layer for `mysqlclient`, but for some other Python package). +- Review `requirements.txt`. You may need to edit this file - it should have the Python package for which you are building an AWS Lambda layer. +- Review the `pip_and_copy.sh` script. You may need to edit this as per your needs (e.g. not building a layer for `mysqlclient`, but for some other Python package). +- Invoke the `build.sh` shell script (e.g. by `bash build.sh`). + +The `build.sh` script will perform all the necessary steps and if successful, will produce a `layer.zip` file in the `build_output` directory. + +`build.sh` will use the `Dockerfile` to build a docker image based off the `lambci/lambda:build-python3.8` image that very-closely replicates the AWS Lambda environment. Any build dependencies (e.g. RPM packages needed in the build environment) should be specified in the `Dockerfile` beforehand. + +After the docker image has been built, `build.sh` runs `pip_and_copy.sh` which in turn runs `pip install -r requirements.txt` and copies over the necessary `.so` file to the output directory. Finally, `build.sh` zips up the build artifacts in the `build_output/python` and `build_output/lib` directories into a zip ready for upload. This `layer.zip` file is the final artifact, ready-to-upload to AWS Lambda for your new layer. -`build.sh` will use the `Dockerfile` to build a docker image based off the `lambci/lambda:build-python3.8` image that very-closely replicates the AWS Lambda environment. The `mysql-community-devel` RPM will be downloaded and installed in the image. This is necessary to `pip install mysqlclient` in Amazon Linux 2. After `pip install mysqlclient` in the docker image has succeeded, the correct `.so` file and the python libs are copied out from the docker container and zipped into `build_output/layer.zip`. +If you are building a layer for `mysqlclient`, `build.sh` specifically does the following: +- Downloads and installs the correct, appropriate `mysql-community-devel` RPM in the docker image. This is necessary to `pip install mysqlclient` in Amazon Linux 2. +- Invokes `pip_and_copy.sh` to `pip install mysqlclient` and copy the correct `.so` file and the python libs out from the docker container and into the `build_output/python` directory. +- Zips the `build_output/python` and `build_output/lib` dir into `build_output/layer.zip`. -This `layer.zip` file is the final artifact, ready-to-upload to AWS Lambda for your new layer. +### full example for building a layer for ANY Python package -### create a new AWS layer with `layer.zip` +Check out the `cchardet-wheel` branch and view `Dockerfile` and `pip_and_copy.sh` files. This branch was created to build the [cchardet](https://github.com/PyYoshi/cChardet) Python package (https://pypi.org/project/cchardet) for AWS Lambda. This is a different procedure from how `mysqlclient` is built. + + +## Create a new AWS layer with `layer.zip` You should upload `layer.zip` as-is; you don't need to zip or unzip anything. @@ -110,13 +150,34 @@ The easiest way is to use the AWS Lambda web console. Of course, there are many For a nice blog post with screenshots on how to upload a zip file as a new layer to AWS Lambda, read the [Deploying our Layer](https://www.freecodecamp.org/news/lambda-layers-2f80b9211318/) section. -### reference the newly-created layer in your lambda function +## Reference the newly-created layer in your lambda function + +Here's how to add your new layer to your Lambda function's configuration. + +In the AWS Lambda Web Console: + +### step 1 + +Go to the function configuration, click `Layers` and then click the `Add a layer` button. + +![AddLayerStep1](images/add-layer-to-lambda-config/AddLambdaLayer-1.png) + +### step 2 + +On the `Add Layer` page, select `Custom layers`, then select the exact custom layer in the dropdown. + +Next, select the version of the custom layer. If this is a brand-new layer, it will only have one version: select version `1`. + +Click the `Add` button. + +![AddLayerStep2](images/add-layer-to-lambda-config/AddLambdaLayer-2.png) -Add the new layer to your Lambda function's configuration. +### step 3 -[TODO: ADD SCREENSHOTS HERE] +Finally, confirm that the custom layer has been added to your Lambda function as shown here: +![AddLayerStep3](images/add-layer-to-lambda-config/AddLambdaLayer-3.png) -### import and run! +## Import and run! Finally, you can simply `import MySQLdb` in your Python3 Lambda function; here's a barebones example: @@ -132,15 +193,15 @@ def lambda_handler(event, context): If you get the success message and don't see an error like `ModuleNotFoundError: No module named 'MySQLdb'` or `ImportError: No module named _mysql`, then you're all set to use `mysqlclient` on AWS Lambda. -## Feedback and Contributions +# Feedback and Contributions ... are most welcome. Please file PRs and issues as you see fit. Will respond to them as soon as possible. -## Troubleshooting +# Troubleshooting See the [`mysqlclient` FAQ](https://github.com/PyMySQL/mysqlclient-python/blob/a33e1c38363b8c71775394965ca70d576ffd3a90/doc/FAQ.rst) that covers troubleshooting for common error cases, including build errors. -## Credits and Thanks +# Credits and Thanks The work in this repo is largely based off Seungyeon Kim(Acuros Kim)'s project at: https://github.com/StyleShare/aws-lambda-python3-mysql - thanks! diff --git a/build.sh b/build.sh index 902c8e9..655364b 100755 --- a/build.sh +++ b/build.sh @@ -3,9 +3,9 @@ PKG_DIR='build_output/python' LIB_DIR='build_output/lib' # set the docker image name here (optional) -IMAGE_NAME='nonbeing/lambda-python38-mysqlclient' +IMAGE_NAME='nonbeing/lambda-python38-boto3' -sudo rm -rf ${PKG_DIR} ${LIB_DIR} +sudo rm -rf build_output mkdir -p ${PKG_DIR} && mkdir -p ${LIB_DIR} # build a docker image closely matching the AWS Lambda environment, with mysql-devel installed @@ -18,4 +18,4 @@ if [ $? -eq 0 ]; then zip -r layer.zip . else echo "[ERROR] Docker build failed! Not building `layer.zip` file. Abort." -fi \ No newline at end of file +fi diff --git a/build_output/layer.zip b/build_output/layer.zip index 94d3ada..eb9c240 100644 Binary files a/build_output/layer.zip and b/build_output/layer.zip differ diff --git a/images/add-layer-to-lambda-config/AddLambdaLayer-1.png b/images/add-layer-to-lambda-config/AddLambdaLayer-1.png new file mode 100644 index 0000000..cfe3996 Binary files /dev/null and b/images/add-layer-to-lambda-config/AddLambdaLayer-1.png differ diff --git a/images/add-layer-to-lambda-config/AddLambdaLayer-2.png b/images/add-layer-to-lambda-config/AddLambdaLayer-2.png new file mode 100644 index 0000000..a75cc39 Binary files /dev/null and b/images/add-layer-to-lambda-config/AddLambdaLayer-2.png differ diff --git a/images/add-layer-to-lambda-config/AddLambdaLayer-3.png b/images/add-layer-to-lambda-config/AddLambdaLayer-3.png new file mode 100644 index 0000000..b78c4cc Binary files /dev/null and b/images/add-layer-to-lambda-config/AddLambdaLayer-3.png differ diff --git a/pip_and_copy.sh b/pip_and_copy.sh index b9da781..536d79a 100755 --- a/pip_and_copy.sh +++ b/pip_and_copy.sh @@ -5,14 +5,14 @@ LIB_DIR=$2 pip install -r requirements.txt -t ${PKG_DIR}; -for i in `ls /usr/lib64/mysql/libmysqlclient.so*`; -do - echo "Checking .so file: '$i'" - if [[ $i =~ libmysqlclient.so.[[:digit:]]+$ ]]; - then - # only copy libmysqlclient.so.21, NOT libmysqlclient.so or libmysqlclient.so.21.1.20 - # because libmysqlclient.so.21 is the necessary and sufficient file for mysqlclient to work - echo "COPYING '$i' to output dir..." - cp $i ${LIB_DIR} - fi -done \ No newline at end of file +# for i in `ls /usr/lib64/mysql/libmysqlclient.so*`; +# do +# echo "Checking .so file: '$i'" +# if [[ $i =~ libmysqlclient.so.[[:digit:]]+$ ]]; +# then +# # only copy libmysqlclient.so.21, NOT libmysqlclient.so or libmysqlclient.so.21.1.20 +# # because libmysqlclient.so.21 is the necessary and sufficient file for mysqlclient to work +# echo "COPYING '$i' to output dir..." +# cp $i ${LIB_DIR} +# fi +# done \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 87cd15a..1a46f92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -mysqlclient \ No newline at end of file +botocore +boto3