gdal/docker/util.sh

485 строки
17 KiB
Bash
Исходник Обычный вид История

#!/bin/bash
# This file is available at the option of the licensee under:
# Public domain
# or licensed under MIT (LICENSE.TXT) Copyright 2019 Even Rouault <even.rouault@spatialys.com>
set -e
if test "${SCRIPT_DIR}" = ""; then
echo "SCRIPT_DIR not defined"
exit 1
fi
if test "${TARGET_IMAGE}" = ""; then
echo "TARGET_IMAGE not defined"
exit 1
fi
usage()
{
echo "Usage: build.sh [--push] [--docker-repository repo] [--tag name] [--gdal tag|sha1|master] [--gdal-repository repo] [--proj tag|sha1|master] [--release]"
# Non-documented: --test-python
echo ""
echo "--push: push image to docker repository (defaults to ghcr.io)"
echo "--tag name: suffix to append to image name. Defaults to 'latest' for non release builds or the GDAL tag name for release builds"
echo "--gdal tag|sha1|master: GDAL version to use. Defaults to master"
echo "--proj tag|sha1|master: PROJ version to use. Defaults to master"
echo "--gdal-repository repo: github repository. Defaults to OSGeo/gdal"
echo "--release. Whether this is a release build. In which case --gdal tag must be used."
echo "--with-debug-symbols/--without-debug-symbols. Whether to include debug symbols. Only applies to ubuntu-full, default is to include for non-release builds."
exit 1
}
RELEASE=no
ARCH_PLATFORMS="linux/amd64"
DOCKER_REPO=ghcr.io
GDAL_REPOSITORY=OSGeo/gdal
while (( "$#" ));
do
case "$1" in
-h|--help)
usage
;;
--push)
PUSH_GDAL_DOCKER_IMAGE=yes
shift
;;
docker-repository)
shift
DOCKER_REPO="$1"
shift
;;
--gdal)
shift
GDAL_VERSION="$1"
shift
;;
--gdal-repository)
shift
GDAL_REPOSITORY="$1"
shift
;;
--platform)
shift
ARCH_PLATFORMS="$1"
shift
;;
--proj)
shift
PROJ_VERSION="$1"
shift
;;
--tag)
shift
TAG_NAME="$1"
shift
;;
--release)
RELEASE=yes
shift
;;
--with-debug-symbols)
WITH_DEBUG_SYMBOLS=yes
shift
;;
--with-multi-arch)
DOCKER_BUILDKIT=1
DOCKER_CLI_EXPERIMENTAL=enabled
shift
;;
--without-debug-symbols)
WITH_DEBUG_SYMBOLS=no
shift
;;
--test-python)
TEST_PYTHON=yes
shift
;;
# Unknown option
*)
echo "Unrecognized option: $1"
usage
;;
esac
done
echo "${DOCKER_REPO}" > /tmp/gdal_docker_repo.txt
if test "${DOCKER_BUILDKIT}" = "1" && test "${DOCKER_CLI_EXPERIMENTAL}" = "enabled"; then
DOCKER_BUILDX="buildx"
DOCKER_BUILDX_ARGS=("--platform" "${ARCH_PLATFORMS}")
fi
# Docker 23 uses BuildKit by default on Linux, but BuildKit prevents
# custom networks using docker build --network, so disable BuildKit for now.
# https://github.com/moby/buildkit/issues/978
if test -z "${DOCKER_BUILDKIT}"; then
export DOCKER_BUILDKIT=0
fi
if test "${RELEASE}" = "yes"; then
if test "${GDAL_VERSION}" = ""; then
echo "--gdal tag must be specified when --release is used."
exit 1
fi
if test "${GDAL_VERSION}" = "master"; then
echo "--gdal master not allowed when --release is used."
exit 1
fi
if test "${PROJ_VERSION}" = ""; then
echo "--proj tag|sha1|master must be specified when --release is used."
exit 1
fi
if test "${TAG_NAME}" = ""; then
TAG_NAME="${GDAL_VERSION}"
fi
if test "${WITH_DEBUG_SYMBOLS}" = ""; then
WITH_DEBUG_SYMBOLS=no
fi
else
if test "${TAG_NAME}" = ""; then
TAG_NAME=latest
fi
if test "${WITH_DEBUG_SYMBOLS}" = ""; then
WITH_DEBUG_SYMBOLS=yes
fi
fi
check_image()
{
TMP_IMAGE_NAME="$1"
docker run --rm "${TMP_IMAGE_NAME}" gdalinfo --version
docker run --rm "${TMP_IMAGE_NAME}" projinfo EPSG:4326
if test "${TEST_PYTHON}" != ""; then
docker run --rm "${TMP_IMAGE_NAME}" python3 -c "from osgeo import gdal, gdalnumeric; print(gdal.VersionInfo(''))"
fi
}
# No longer used
cleanup_rsync()
{
rm -f "${RSYNC_DAEMON_TEMPFILE}"
if test "${RSYNC_PID}" != ""; then
kill "${RSYNC_PID}" || true
fi
}
# No longer used
trap_error_exit()
{
echo "Exit on error... clean up"
cleanup_rsync
exit 1
}
# No longer used
start_rsync_host()
{
# If rsync is available then start it as a temporary daemon
if test "${USE_CACHE:-yes}" = "yes" -a -x "$(command -v rsync)"; then
RSYNC_SERVER_IP=$(ip -4 -o addr show docker0 | awk '{print $4}' | cut -d "/" -f 1)
if test "${RSYNC_SERVER_IP}" = ""; then
exit 1
fi
RSYNC_DAEMON_TEMPFILE=$(mktemp)
# Trap exit
trap "trap_error_exit" EXIT
cat <<EOF > "${RSYNC_DAEMON_TEMPFILE}"
[gdal-docker-cache]
path = $HOME/gdal-docker-cache
comment = GDAL Docker cache
hosts allow = ${RSYNC_SERVER_IP}/24
use chroot = false
read only = false
EOF
RSYNC_PORT=23985
while true; do
rsync --port=${RSYNC_PORT} --address="${RSYNC_SERVER_IP}" --config="${RSYNC_DAEMON_TEMPFILE}" --daemon --no-detach &
RSYNC_PID=$!
sleep 1
kill -0 ${RSYNC_PID} 2>/dev/null && break
echo "Port ${RSYNC_PORT} is in use. Trying next one"
RSYNC_PORT=$((RSYNC_PORT+1))
done
echo "rsync daemon forked as process ${RSYNC_PID} listening on port ${RSYNC_PORT}"
RSYNC_REMOTE="rsync://${RSYNC_SERVER_IP}:${RSYNC_PORT}/gdal-docker-cache/${TARGET_IMAGE}"
mkdir -p "$HOME/gdal-docker-cache/${TARGET_IMAGE}/proj"
mkdir -p "$HOME/gdal-docker-cache/${TARGET_IMAGE}/gdal"
mkdir -p "$HOME/gdal-docker-cache/${TARGET_IMAGE}/spatialite"
else
RSYNC_REMOTE=""
fi
}
# No longer used
stop_rsync_host()
{
if test "${RSYNC_REMOTE}" != ""; then
cleanup_rsync
trap - EXIT
fi
}
build_cmd()
{
if test "${DOCKER_BUILDX}" = "buildx"; then
echo "${DOCKER_BUILDX}" build "${DOCKER_BUILDX_ARGS[@]}"
else
echo build
fi
}
if test "${TARGET_IMAGE}" = "osgeo/gdal:ubuntu-full"; then
PROJ_DATUMGRID_LATEST_LAST_MODIFIED=$(curl -Is https://cdn.proj.org/index.html | grep -i Last-Modified)
else
PROJ_DATUMGRID_LATEST_LAST_MODIFIED=$(curl -Is http://download.osgeo.org/proj/proj-datumgrid-latest.zip | grep -i Last-Modified)
fi
echo "Using PROJ_DATUMGRID_LATEST_LAST_MODIFIED=${PROJ_DATUMGRID_LATEST_LAST_MODIFIED}"
if test "${PROJ_VERSION}" = "" -o "${PROJ_VERSION}" = "master"; then
PROJ_VERSION=$(curl -Ls https://api.github.com/repos/OSGeo/PROJ/commits/HEAD -H "Accept: application/vnd.github.VERSION.sha")
fi
echo "Using PROJ_VERSION=${PROJ_VERSION}"
if test "${GDAL_VERSION}" = "" -o "${GDAL_VERSION}" = "master"; then
GDAL_VERSION=$(curl -Ls "https://api.github.com/repos/${GDAL_REPOSITORY}/commits/HEAD" -H "Accept: application/vnd.github.VERSION.sha")
fi
echo "Using GDAL_VERSION=${GDAL_VERSION}"
echo "Using GDAL_REPOSITORY=${GDAL_REPOSITORY}"
IMAGE_NAME="${TARGET_IMAGE}-${TAG_NAME}"
REPO_IMAGE_NAME="${DOCKER_REPO}/${IMAGE_NAME}"
BUILDER_IMAGE_NAME="${DOCKER_REPO}/${IMAGE_NAME}_builder"
if test "${RELEASE}" = "yes"; then
BUILD_ARGS=(
"--build-arg" "PROJ_DATUMGRID_LATEST_LAST_MODIFIED=${PROJ_DATUMGRID_LATEST_LAST_MODIFIED}" \
"--build-arg" "PROJ_VERSION=${PROJ_VERSION}" \
"--build-arg" "GDAL_VERSION=${GDAL_VERSION}" \
"--build-arg" "GDAL_REPOSITORY=${GDAL_REPOSITORY}" \
"--build-arg" "GDAL_BUILD_IS_RELEASE=YES" \
"--build-arg" "WITH_DEBUG_SYMBOLS=${WITH_DEBUG_SYMBOLS}" \
)
if test "${BASE_IMAGE}" != ""; then
BUILD_ARGS+=("--build-arg" "BASE_IMAGE=${BASE_IMAGE}")
if test "${TARGET_IMAGE}" = "osgeo/gdal:ubuntu-full" -o "${TARGET_IMAGE}" = "osgeo/gdal:ubuntu-small"; then
BUILD_ARGS+=("--build-arg" "TARGET_BASE_IMAGE=${BASE_IMAGE}")
fi
fi
LABEL_ARGS=(
"--label" "org.opencontainers.image.description=GDAL is an open source MIT licensed translator library for raster and vector geospatial data formats." \
"--label" "org.opencontainers.image.title=GDAL ${TARGET_IMAGE}" \
"--label" "org.opencontainers.image.licenses=MIT" \
"--label" "org.opencontainers.image.source=https://github.com/${GDAL_REPOSITORY}" \
"--label" "org.opencontainers.image.url=https://github.com/${GDAL_REPOSITORY}" \
"--label" "org.opencontainers.image.revision=${GDAL_VERSION}" \
"--label" "org.opencontainers.image.version=${TAG_NAME}" \
)
if test "${DOCKER_BUILDX}" = "buildx" -a "${PUSH_GDAL_DOCKER_IMAGE}" = "yes"; then
docker $(build_cmd) "${BUILD_ARGS[@]}" "${LABEL_ARGS[@]}" -t "${REPO_IMAGE_NAME}" --push "${SCRIPT_DIR}"
else
docker $(build_cmd) "${BUILD_ARGS[@]}" "${LABEL_ARGS[@]}" --target builder -t "${BUILDER_IMAGE_NAME}" "${SCRIPT_DIR}"
docker $(build_cmd) "${BUILD_ARGS[@]}" "${LABEL_ARGS[@]}" -t "${REPO_IMAGE_NAME}" "${SCRIPT_DIR}"
if test "${DOCKER_BUILDX}" != "buildx"; then
check_image "${REPO_IMAGE_NAME}"
fi
if test "${PUSH_GDAL_DOCKER_IMAGE}" = "yes"; then
docker push "${REPO_IMAGE_NAME}"
fi
fi
else
IMAGE_NAME_WITH_ARCH="${REPO_IMAGE_NAME}"
if test "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-full-latest" \
-o "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-small-latest" \
-o "${IMAGE_NAME}" = "osgeo/gdal:alpine-small-latest" \
-o "${IMAGE_NAME}" = "osgeo/gdal:alpine-normal-latest"; then
if test "${DOCKER_BUILDX}" != "buildx"; then
ARCH_PLATFORM_ARCH=$(echo ${ARCH_PLATFORMS} | sed "s/linux\///")
IMAGE_NAME_WITH_ARCH="${REPO_IMAGE_NAME}-${ARCH_PLATFORM_ARCH}"
BUILDER_IMAGE_NAME="${BUILDER_IMAGE_NAME}_${ARCH_PLATFORM_ARCH}"
fi
fi
OLD_BUILDER_ID=$(docker image ls "${BUILDER_IMAGE_NAME}" -q)
OLD_IMAGE_ID=$(docker image ls "${IMAGE_NAME_WITH_ARCH}" -q)
if test "${GDAL_RELEASE_DATE}" = ""; then
GDAL_RELEASE_DATE=$(date "+%Y%m%d")
fi
echo "Using GDAL_RELEASE_DATE=${GDAL_RELEASE_DATE}"
RSYNC_DAEMON_CONTAINER=gdal_rsync_daemon
BUILD_NETWORK=docker_build_gdal
HOST_CACHE_DIR="$HOME/gdal-docker-cache"
mkdir -p "${HOST_CACHE_DIR}/${TARGET_IMAGE}/proj/x86_64"
mkdir -p "${HOST_CACHE_DIR}/${TARGET_IMAGE}/proj/aarch64"
mkdir -p "${HOST_CACHE_DIR}/${TARGET_IMAGE}/gdal/x86_64"
mkdir -p "${HOST_CACHE_DIR}/${TARGET_IMAGE}/gdal/aarch64"
mkdir -p "${HOST_CACHE_DIR}/${TARGET_IMAGE}/spatialite"
# Start a Docker container that has a rsync daemon, mounting HOST_CACHE_DIR
if ! docker ps | grep "${RSYNC_DAEMON_CONTAINER}"; then
RSYNC_DAEMON_IMAGE=osgeo/gdal:gdal_rsync_daemon
docker rmi "${RSYNC_DAEMON_IMAGE}" 2>/dev/null || true
docker $(build_cmd) -t "${RSYNC_DAEMON_IMAGE}" - <<EOF
FROM alpine
VOLUME /opt/gdal-docker-cache
EXPOSE 23985
RUN apk add --no-cache rsync \
&& mkdir -p /opt/gdal-docker-cache \
&& echo "[gdal-docker-cache]" > /etc/rsyncd.conf \
&& echo "path = /opt/gdal-docker-cache" >> /etc/rsyncd.conf \
&& echo "hosts allow = *" >> /etc/rsyncd.conf \
&& echo "read only = false" >> /etc/rsyncd.conf \
&& echo "use chroot = false" >> /etc/rsyncd.conf
CMD rsync --daemon --port 23985 && while sleep 1; do true; done
EOF
if ! docker network ls | grep "${BUILD_NETWORK}"; then
docker network create "${BUILD_NETWORK}"
fi
THE_UID=$(id -u "${USER}")
THE_GID=$(id -g "${USER}")
docker run -d -u "${THE_UID}:${THE_GID}" --rm \
-v "${HOST_CACHE_DIR}":/opt/gdal-docker-cache \
--name "${RSYNC_DAEMON_CONTAINER}" \
--network "${BUILD_NETWORK}" \
--network-alias "${RSYNC_DAEMON_CONTAINER}" \
"${RSYNC_DAEMON_IMAGE}"
fi
RSYNC_REMOTE="rsync://${RSYNC_DAEMON_CONTAINER}:23985/gdal-docker-cache/${TARGET_IMAGE}"
BUILD_ARGS=(
"--build-arg" "PROJ_DATUMGRID_LATEST_LAST_MODIFIED=${PROJ_DATUMGRID_LATEST_LAST_MODIFIED}" \
"--build-arg" "PROJ_VERSION=${PROJ_VERSION}" \
"--build-arg" "GDAL_VERSION=${GDAL_VERSION}" \
"--build-arg" "GDAL_REPOSITORY=${GDAL_REPOSITORY}" \
"--build-arg" "GDAL_RELEASE_DATE=${GDAL_RELEASE_DATE}" \
"--build-arg" "RSYNC_REMOTE=${RSYNC_REMOTE}" \
"--build-arg" "WITH_DEBUG_SYMBOLS=${WITH_DEBUG_SYMBOLS}" \
)
if test "${BASE_IMAGE}" != ""; then
BUILD_ARGS+=("--build-arg" "BASE_IMAGE=${BASE_IMAGE}")
if test "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-full-latest" -o "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-small-latest"; then
BUILD_ARGS+=("--build-arg" "TARGET_BASE_IMAGE=${BASE_IMAGE}")
fi
else
if test "${DOCKER_BUILDX}" != "buildx" -a \( "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-full-latest" -o "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-small-latest" \); then
if test "${ARCH_PLATFORMS}" = "linux/arm64"; then
BASE_IMAGE=$(grep "ARG BASE_IMAGE=" ${SCRIPT_DIR}/Dockerfile | sed "s/ARG BASE_IMAGE=//")
echo "Fetching digest for ${BASE_IMAGE} ${ARCH_PLATFORMS}..."
ARCH_PLATFORM_ARCH=$(echo ${ARCH_PLATFORMS} | sed "s/linux\///")
# Below no longer works with recent Ubuntu images with Docker client < 23.0.0
# Cf https://github.com/moby/moby/issues/44898
#TARGET_BASE_IMAGE_DIGEST=$(docker manifest inspect ${BASE_IMAGE} | jq --raw-output '.manifests[] | (if .platform.architecture == "'${ARCH_PLATFORM_ARCH}'" then .digest else empty end)')
# So workaround the issue by pulling explicitly the image
docker pull arm64v8/${BASE_IMAGE}
TARGET_BASE_IMAGE_DIGEST=$(docker inspect arm64v8/${BASE_IMAGE} | jq --raw-output '.[0].Id')
echo "${TARGET_BASE_IMAGE_DIGEST}"
BUILD_ARGS+=("--build-arg" "TARGET_ARCH=${ARCH_PLATFORM_ARCH}")
#BUILD_ARGS+=("--build-arg" "TARGET_BASE_IMAGE=${BASE_IMAGE}@${TARGET_BASE_IMAGE_DIGEST}")
BUILD_ARGS+=("--build-arg" "TARGET_BASE_IMAGE=${TARGET_BASE_IMAGE_DIGEST}")
# echo "${BUILD_ARGS[@]}"
fi
elif test "${DOCKER_BUILDX}" != "buildx" -a \( "${IMAGE_NAME}" = "osgeo/gdal:alpine-small-latest" -o "${IMAGE_NAME}" = "osgeo/gdal:alpine-normal-latest" \); then
if test "${ARCH_PLATFORMS}" = "linux/arm64"; then
ALPINE_VERSION=$(grep "ARG ALPINE_VERSION=" ${SCRIPT_DIR}/Dockerfile | sed "s/ARG ALPINE_VERSION=//")
BASE_IMAGE="alpine:${ALPINE_VERSION}"
echo "Fetching digest for ${BASE_IMAGE} ${ARCH_PLATFORMS}..."
ARCH_PLATFORM_ARCH=$(echo ${ARCH_PLATFORMS} | sed "s/linux\///")
TARGET_BASE_IMAGE_DIGEST=$(docker manifest inspect ${BASE_IMAGE} | jq --raw-output '.manifests[] | (if .platform.architecture == "'${ARCH_PLATFORM_ARCH}'" then .digest else empty end)')
echo "${TARGET_BASE_IMAGE_DIGEST}"
BUILD_ARGS+=("--build-arg" "ALPINE_VERSION=${ALPINE_VERSION}@${TARGET_BASE_IMAGE_DIGEST}")
# echo "${BUILD_ARGS[@]}"
fi
fi
fi
docker $(build_cmd) --network "${BUILD_NETWORK}" "${BUILD_ARGS[@]}" --target builder \
-t "${BUILDER_IMAGE_NAME}" "${SCRIPT_DIR}"
docker $(build_cmd) "${BUILD_ARGS[@]}" -t "${IMAGE_NAME_WITH_ARCH}" "${SCRIPT_DIR}"
if test "${DOCKER_BUILDX}" != "buildx"; then
check_image "${IMAGE_NAME_WITH_ARCH}"
fi
if test "${PUSH_GDAL_DOCKER_IMAGE}" = "yes"; then
if test "${DOCKER_BUILDX}" = "buildx"; then
docker $(build_cmd) "${BUILD_ARGS[@]}" -t "${REPO_IMAGE_NAME}" --push "${SCRIPT_DIR}"
else
docker push "${IMAGE_NAME_WITH_ARCH}"
fi
if test "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-full-latest"; then
if test "${DOCKER_BUILDX}" = "buildx"; then
docker $(build_cmd) "${BUILD_ARGS[@]}" -t "${DOCKER_REPO}/osgeo/gdal:latest" --push "${SCRIPT_DIR}"
else
if test "${ARCH_PLATFORMS}" = "linux/amd64"; then
docker image tag "${IMAGE_NAME_WITH_ARCH}" "${DOCKER_REPO}/osgeo/gdal:latest"
docker push "${DOCKER_REPO}/osgeo/gdal:latest"
fi
fi
fi
fi
# Cleanup previous images
NEW_BUILDER_ID=$(docker image ls "${BUILDER_IMAGE_NAME}" -q)
NEW_IMAGE_ID=$(docker image ls "${IMAGE_NAME_WITH_ARCH}" -q)
if test "${OLD_BUILDER_ID}" != "" -a "${OLD_BUILDER_ID}" != "${NEW_BUILDER_ID}"; then
docker rmi "${OLD_BUILDER_ID}"
fi
if test "${OLD_IMAGE_ID}" != "" -a "${OLD_IMAGE_ID}" != "${NEW_IMAGE_ID}"; then
docker rmi "${OLD_IMAGE_ID}"
fi
if test "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-full-latest" \
-o "${IMAGE_NAME}" = "osgeo/gdal:ubuntu-small-latest" \
-o "${IMAGE_NAME}" = "osgeo/gdal:alpine-small-latest" \
-o "${IMAGE_NAME}" = "osgeo/gdal:alpine-normal-latest"; then
if test "${DOCKER_BUILDX}" != "buildx" -a "${ARCH_PLATFORMS}" = "linux/amd64"; then
docker image tag "${IMAGE_NAME_WITH_ARCH}" "${REPO_IMAGE_NAME}"
fi
fi
fi