After a lot of digging I managed to get this working.
I'm not 100% sure if the cause is the older GnuPG binary installed on the Lambda image by default, but to be sure I decided to build a GnuPG 2.3.1 layer for lambda which I confirmed was working as expected in a Docker container.
I used https://github.com/skeeto/lean-static-gpg/blob/master/build.sh as a foundation for compiling the binary in
Docker, but updated it to include compression, which was required for this use case.
Here is the updated updated build.sh script I used, optimized for building for Lambda:
#!/bin/sh
set -e
MUSL_VERSION=1.2.2
GNUPG_VERSION=2.3.1
LIBASSUAN_VERSION=2.5.5
LIBGCRYPT_VERSION=1.9.2
LIBGPGERROR_VERSION=1.42
LIBKSBA_VERSION=1.5.1
NPTH_VERSION=1.6
PINENTRY_VERSION=1.1.1
BZIP_VERSION=1.0.6-g10
ZLIB_VERSION=1.2.12
DESTDIR=""
PREFIX="/opt"
WORK="$PWD/work"
PATH="$PWD/work/deps/bin:$PATH"
NJOBS=$(nproc)
clean() {
rm -rf "$WORK"
}
distclean() {
clean
rm -rf download
}
download() {
gnupgweb=https://gnupg.org/ftp/gcrypt
mkdir -p download
(
cd download/
xargs -n1 curl -O <<EOF
https://www.musl-libc.org/releases/musl-$MUSL_VERSION.tar.gz
$gnupgweb/gnupg/gnupg-$GNUPG_VERSION.tar.bz2
$gnupgweb/libassuan/libassuan-$LIBASSUAN_VERSION.tar.bz2
$gnupgweb/libgcrypt/libgcrypt-$LIBGCRYPT_VERSION.tar.bz2
$gnupgweb/libgpg-error/libgpg-error-$LIBGPGERROR_VERSION.tar.bz2
$gnupgweb/libksba/libksba-$LIBKSBA_VERSION.tar.bz2
$gnupgweb/npth/npth-$NPTH_VERSION.tar.bz2
$gnupgweb/pinentry/pinentry-$PINENTRY_VERSION.tar.bz2
$gnupgweb/bzip2/bzip2-$BZIP_VERSION.tar.gz
$gnupgweb/zlib/zlib-$ZLIB_VERSION.tar.gz
EOF
)
}
clean
if [ ! -d download/ ]; then
download
fi
mkdir -p "$DESTDIR$PREFIX" "$WORK/deps"
tar -C "$WORK" -xzf download/musl-$MUSL_VERSION.tar.gz
(
mkdir -p "$WORK/musl"
cd "$WORK/musl"
../musl-$MUSL_VERSION/configure \
--prefix="$WORK/deps" \
--enable-wrapper=gcc \
--syslibdir="$WORK/deps/lib"
make -kj$NJOBS
make install
make clean
)
tar -C "$WORK" -xzf download/zlib-$ZLIB_VERSION.tar.gz
(
mkdir -p "$WORK/zlib"
cd "$WORK/zlib"
../zlib-$ZLIB_VERSION/configure \
--prefix="$WORK/deps"
make -kj$NJOBS
make install
make clean
)
tar -C "$WORK" -xzf download/bzip2-$BZIP_VERSION.tar.gz
(
export CFLAGS="-fPIC"
cd "$WORK/bzip2-$BZIP_VERSION"
make install PREFIX="$WORK/deps"
make clean
)
tar -C "$WORK" -xjf download/npth-$NPTH_VERSION.tar.bz2
(
mkdir -p "$WORK/npth"
cd "$WORK/npth"
../npth-$NPTH_VERSION/configure \
CC="$WORK/deps/bin/musl-gcc" \
--prefix="$WORK/deps" \
--enable-shared=no \
--enable-static=yes
make -kj$NJOBS
make install
)
tar -C "$WORK" -xjf download/libgpg-error-$LIBGPGERROR_VERSION.tar.bz2
(
mkdir -p "$WORK/libgpg-error"
cd "$WORK/libgpg-error"
../libgpg-error-$LIBGPGERROR_VERSION/configure \
CC="$WORK/deps/bin/musl-gcc" \
--prefix="$WORK/deps" \
--enable-shared=no \
--enable-static=yes \
--disable-nls \
--disable-doc \
--disable-languages
make -kj$NJOBS
make install
)
tar -C "$WORK" -xjf download/libassuan-$LIBASSUAN_VERSION.tar.bz2
(
mkdir -p "$WORK/libassuan"
cd "$WORK/libassuan"
../libassuan-$LIBASSUAN_VERSION/configure \
CC="$WORK/deps/bin/musl-gcc" \
--prefix="$WORK/deps" \
--enable-shared=no \
--enable-static=yes \
--with-libgpg-error-prefix="$WORK/deps"
make -kj$NJOBS
make install
)
tar -C "$WORK" -xjf download/libgcrypt-$LIBGCRYPT_VERSION.tar.bz2
(
mkdir -p "$WORK/libgcrypt"
cd "$WORK/libgcrypt"
../libgcrypt-$LIBGCRYPT_VERSION/configure \
CC="$WORK/deps/bin/musl-gcc" \
--prefix="$WORK/deps" \
--enable-shared=no \
--enable-static=yes \
--disable-doc \
--with-libgpg-error-prefix="$WORK/deps"
make -kj$NJOBS
make install
)
tar -C "$WORK" -xjf download/libksba-$LIBKSBA_VERSION.tar.bz2
(
mkdir -p "$WORK/libksba"
cd "$WORK/libksba"
../libksba-$LIBKSBA_VERSION/configure \
CC="$WORK/deps/bin/musl-gcc" \
--prefix="$WORK/deps" \
--enable-shared=no \
--enable-static=yes \
--with-libgpg-error-prefix="$WORK/deps"
make -kj$NJOBS
make install
)
tar -C "$WORK" -xjf download/gnupg-$GNUPG_VERSION.tar.bz2
(
mkdir -p "$WORK/gnupg"
cd "$WORK/gnupg"
../gnupg-$GNUPG_VERSION/configure \
CC="$WORK/deps/bin/musl-gcc" \
LDFLAGS="-static -s" \
--prefix="$PREFIX" \
--with-libgpg-error-prefix="$WORK/deps" \
--with-libgcrypt-prefix="$WORK/deps" \
--with-libassuan-prefix="$WORK/deps" \
--with-ksba-prefix="$WORK/deps" \
--with-npth-prefix="$WORK/deps" \
--with-agent-pgm="$PREFIX/bin/gpg-agent" \
--with-pinentry-pgm="$PREFIX/bin/pinentry" \
--enable-zip \
--enable-bzip2 \
--disable-card-support \
--disable-ccid-driver \
--disable-dirmngr \
--disable-gnutls \
--disable-gpg-blowfish \
--disable-gpg-cast5 \
--disable-gpg-idea \
--disable-gpg-md5 \
--disable-gpg-rmd160 \
--disable-gpgtar \
--disable-ldap \
--disable-libdns \
--disable-nls \
--disable-ntbtls \
--disable-photo-viewers \
--disable-scdaemon \
--disable-sqlite \
--disable-wks-tools
make -kj$NJOBS
make install DESTDIR="$DESTDIR"
rm "$DESTDIR$PREFIX/bin/gpgscm"
)
tar -C "$WORK" -xjf download/pinentry-$PINENTRY_VERSION.tar.bz2
(
mkdir -p "$WORK/pinentry"
cd "$WORK/pinentry"
../pinentry-$PINENTRY_VERSION/configure \
CC="$WORK/deps/bin/musl-gcc" \
LDFLAGS="-static -s" \
--prefix="$PREFIX" \
--with-libgpg-error-prefix="$WORK/deps" \
--with-libassuan-prefix="$WORK/deps" \
--disable-ncurses \
--disable-libsecret \
--enable-pinentry-tty \
--disable-pinentry-curses \
--disable-pinentry-emacs \
--disable-inside-emacs \
--disable-pinentry-gtk2 \
--disable-pinentry-gnome3 \
--disable-pinentry-qt \
--disable-pinentry-tqt \
--disable-pinentry-fltk
make -kj$NJOBS
make install DESTDIR="$DESTDIR"
)
rm -rf "$DESTDIR$PREFIX/sbin"
rm -rf "$DESTDIR$PREFIX/share/doc"
rm -rf "$DESTDIR$PREFIX/share/info"
# cleanup
distclean
Below is the Dockerfile used to build the layer:
FROM public.ecr.aws/lambda/python:3.8
# the output volume to extract the build contents
VOLUME ["/opt/bin"]
RUN yum -y groupinstall 'Development Tools'
RUN yum -y install tar gzip zlib bzip2 file hostname
WORKDIR /opt
# copy the build script
COPY static-gnupg-build.sh .
# run the build script
RUN bash ./static-gnupg-build.sh
# when run output the version
ENTRYPOINT [ "/opt/bin/gpg", "--version" ]
Once the code is compiled in the image I copied it to my local directory, zipped it, and published the layer:
docker cp MY_DOCKER_ID:/opt/bin ./gnupg
cd ./gnupg && zip -r gnupg-layer.zip bin
To publish the layer:
aws lambda publish-layer-version \
--layer-name gnupg \
--zip-file fileb://layer-gpg2.3.zip \
--compatible-architectures python3.8
I decided to not use the python-gnupg package to have more control over the exact GnuPG binary flags so I added my own
binary wrapper function:
def gpg_run(flags: list, subprocess_kwargs: dict):
gpg_bin_args = [
'/opt/bin/gpg',
'--no-tty',
'--yes', # don't prompt for input
'--always-trust', # always trust
'--status-fd', '1', # return status to stdout
'--homedir', '/tmp'
]
gpg_bin_args.extend(flags)
print('running cmd', ' '.join(gpg_bin_args))
result = subprocess.run(gpg_bin_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**subprocess_kwargs)
return result.returncode, \
result.stdout.decode('utf-8').split('/n'), \
result.stderr.decode('utf-8').split('/n')
And then added an import key and decode function:
def gpg_import_keys(input):
return gpg_run(flags=['--import'], subprocess_kwargs={input: input})
def gpg_decrypt(input, output):
return gpg_run(flags=['--output', output, '--decrypt', input])
And updated the relavent Lambda code with:
# ...
encrypted_path = '/tmp/encrypted.zip'
decrypted_path = '/tmp/decrypted.zip'
#...
# TODO: import the keys only needs to run once per instance
# ideally would be moved to a singleton
code, stdout, stderr = gpg_import_keys(bytes(MY_KEY_DATA, 'utf-8'))
if code > 0:
raise Exception(f'gpg_import_keys failed with code {code}: {stdout} {stderr}')
print('import_keys stdout =', stdout)
print('import_keys stderr =', stderr)
# Perform decryption.
print('Performing Decryption of', encrypted_path)
code, stdout, stderr = gpg_decrypt(encrypted_path, output=decrypted_path)
if code > 0:
raise Exception(f'gpg_decrypt failed with code {code}: {stderr}')
print('decrypt stdout =', stdout)
print('decrypt stderr =', stderr)
print('Status: OK')
print(decrypted_path, "exists :", Path(decrypted_path).exists())
And now the Cloudwatch log output is as expected and I've confirmed the decoded file is correct!
...
2022-11-17T09:25:22.732-06:00 running cmd:['/opt/bin/gpg', '--no-tty', '--batch', '--yes', '--always-trust', '--status-fd', '1', '--homedir', '/tmp', '--import']
2022-11-17T09:25:22.769-06:00 import_keys ok = True
2022-11-17T09:25:22.769-06:00 import_keys stdout = ['[GNUPG:] IMPORT_OK 0 XXX', '[GNUPG:] KEY_CONSIDERED XXX 0', '[GNUPG:] IMPORT_OK 16 XXX', '[GNUPG:] IMPORT_RES 1 0 0 0 1 0 0 0 0 1 0 1 0 0 0', '']
2022-11-17T09:25:22.769-06:00 import_keys stderr = ['']
2022-11-17T09:25:22.769-06:00 Performing Decryption of /tmp/test.txt.gpg
2022-11-17T09:25:22.769-06:00 running cmd: /opt/bin/gpg --no-tty --yes --always-trust --status-fd 1 --homedir /tmp --output /tmp/decrypted.zip --decrypt /tmp/encrypted.zip
2022-11-17T09:25:22.850-06:00 decrypt stdout = ['[GNUPG:] ENC_TO XXX 1 0', '[GNUPG:] KEY_CONSIDERED XXX 0', '[GNUPG:] DECRYPTION_KEY XXX -', '[GNUPG:] BEGIN_DECRYPTION', '[GNUPG:] DECRYPTION_INFO 0 9 2', '[GNUPG:] PLAINTEXT 62 1667796554 encrypted.zip', '[GNUPG:] PLAINTEXT_LENGTH 428', '[GNUPG:] DECRYPTION_OKAY', '[GNUPG:] GOODMDC', '[GNUPG:] END_DECRYPTION', '']
2022-11-17T09:25:22.850-06:00 decrypt stderr = ['gpg: encrypted with rsa2048 key, ID XXX, created 2022-11-07', ' "XXX"', '']
2022-11-17T09:25:22.850-06:00 /tmp/decrypted.zip exists: True
...
