Merge pull request #425 from neilcook/luajit

Use OpenResty Luajit Package with wforce

.github/workflows/builder.yml

@@ -19,10 +19,10 @@ jobs:
           - ol-8
           - el-9
           - amazon-2
-          - debian-buster
           - debian-bullseye
+          - debian-bookworm
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # for correct version numbers
           submodules: recursive

.github/workflows/docker.yml

@@ -10,7 +10,7 @@ jobs:
     name: docker_test
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           submodules: recursive

.github/workflows/regression_reusable.yml

@@ -14,7 +14,7 @@ jobs:
     name: regression-${{ inputs.compiler }}
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 5
           submodules: recursive

builder-support/debian/compat

@@ -1 +1 @@
-9
+10

builder-support/debian/control

@@ -10,10 +10,10 @@ Build-Depends: cdbs,
                dh-autoreconf,
                libboost-all-dev (>= 1.4.0),
                libcurl4-openssl-dev,
+               wforce-lua-dist,
                libgetdns-dev,
                libhiredis-dev,
                libmaxminddb-dev,
-               libluajit-5.1-dev,
                libprotobuf-dev,
                libreadline-dev,
                libssl-dev,

builder-support/dockerfiles/Dockerfile.debbuild

@@ -1,39 +1,79 @@
-FROM dist-base as package-builder
+FROM dist-base as package-builder-base
 ARG APT_URL
 RUN apt-get -y install devscripts dpkg-dev build-essential wget curl \
                        python3 python3-pip python3-setuptools \
-                       libjsoncpp-dev uuid-dev libz-dev libssl-dev
+                       libjsoncpp-dev uuid-dev libz-dev libssl-dev libldap2-dev
 
 RUN mkdir -p /dist /wforce
 ADD builder/helpers/ /wforce/builder/helpers/
 
 # Used for -p option to only build specific spec files
 ARG BUILDER_PACKAGE_MATCH
 
-@IF [ ! -z "$M_all$M_wforce" ]
 # Build weakforce
 WORKDIR /wforce
 
-ARG BUILDER_VERSION
-ARG BUILDER_RELEASE
-ARG BUILDER_EPOCH
+##############################################################################
+# Separate build stage for luaenv to speedup builds and improve caching
+
+FROM package-builder-base as luaenv-builder
+
+@IF [ ! -z "$M_all$M_lua" ]
+# These are needed for the luaenv build
+@INCLUDE ../../luaenv/Dockerfile.paths-include
+# Build Lua dist in docker, and later only package it in an RPM spec
+# This is easier to debug than build failures in a deb build and gets cached in a layer
+RUN cd /weakforced/luaenv && ./build.sh -p /usr/share/wforce-lua-dist
+@ENDIF
+
+##############################################################################
+# Separate build stage for wforce static libs to speedup builds and improve caching
+
+FROM package-builder-base as wforce-static-libs-builder
 
 COPY --from=sdist /sdist/ /sdist/
 
 RUN wget https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-Linux-x86_64.sh
 RUN sh cmake-3.23.2-Linux-x86_64.sh --skip-license --prefix=/usr
+
 RUN tar xvf /sdist/prometheus-cpp*Source.tar.gz
 RUN mv prometheus-cpp*Source prometheus-cpp
 RUN cd prometheus-cpp/_build && make install
 
 RUN git clone https://github.com/drogonframework/drogon.git
 RUN cd drogon && git checkout tags/v1.9.1 -b v1.9.1
-RUN cd drogon && git submodule init && git submodule update && mkdir _build && cd _build && cmake .. -DBUILD_REDIS=OFF -DBUILD_ORM=OFF -DCMAKE_BUILD_TYPE=Release && make && make install
+RUN cd drogon && git submodule init && git submodule update && mkdir _build && cd _build && cmake --install-prefix=/usr/local .. -DBUILD_REDIS=OFF -DBUILD_ORM=OFF -DCMAKE_BUILD_TYPE=Release && make && make install
+
+##############################################################################
+# Our package-builder target image
+
+FROM package-builder-base as package-builder
+
+ARG BUILDER_VERSION
+ARG BUILDER_RELEASE
+ARG BUILDER_EPOCH
+
+COPY --from=sdist /sdist/ /sdist/
+
+#
+# Lua dist and modules
+#
+@IF [ ! -z "$M_all$M_lua" ]
+RUN mkdir -p luajit-${BUILDER_VERSION}/usr/share/wforce-lua-dist/debian/tmp
+COPY --from=luaenv-builder /usr/share/wforce-lua-dist wforce-lua-dist-${BUILDER_VERSION}/wforce-lua-dist
+# Build and install Lua dist package
+COPY builder-support/luajit-debian wforce-lua-dist-${BUILDER_VERSION}/debian
+# QA_RPATHS for: ERROR 0001: file '/usr/share/wforce-lua-dist/lib/lua/5.1/system/core.so' contains a standard rpath '/usr/lib64' in [/usr/lib64]
+RUN ulimit -n 2000 && QA_RPATHS=1 builder/helpers/build-debs.sh wforce-lua-dist-${BUILDER_VERSION}
+RUN ulimit -n 2000 && apt install -y ./wforce-lua-dist*.deb
+@ENDIF
+
+@IF [ ! -z "$M_all$M_wforce" ]
+COPY --from=wforce-static-libs-builder /usr/local /usr/local
 
 RUN tar xvf /sdist/wforce-${BUILDER_VERSION}.tar.bz2
 COPY builder-support/debian wforce-${BUILDER_VERSION}/debian
 RUN builder/helpers/build-debs.sh wforce-$BUILDER_VERSION
-
 @ENDIF
 
 RUN mv wforce-${BUILDER_VERSION}/wforce*.deb /dist

builder-support/dockerfiles/Dockerfile.debtest

@@ -0,0 +1,15 @@
+# Install the built rpms and test them
+FROM dist-base as dist
+# If you want to install extra packages or do generic configuration,
+# do it before the COPY. Either here, or in the dist-base layer.
+
+COPY --from=sdist /sdist /sdist
+COPY --from=package-builder /dist /dist
+
+# Install built packages with dependencies
+RUN apt install -y /dist/*.deb
+
+# Installation tests
+COPY builder-support/install-tests /wforce/builder-support/install-tests
+WORKDIR /wforce
+RUN builder-support/install-tests/test-simple-exec.sh

builder-support/dockerfiles/Dockerfile.rpmbuild

@@ -1,4 +1,4 @@
-FROM dist-base as package-builder
+FROM dist-base as package-builder-base
 RUN yum install -y rpm-build rpmdevtools python3-rpm-macros wget \
     /usr/bin/python3 /usr/bin/pip3 && \
     yum groupinstall -y "Development Tools"
@@ -13,31 +13,51 @@ ARG CXX=clang++
 RUN mkdir /dist /wforce
 WORKDIR /wforce
 
-# Used for -p option to only build specific spec files
-ARG BUILDER_PACKAGE_MATCH
+##############################################################################
+# Reusable base with extra packages
 
-ARG BUILDER_VERSION
-ARG BUILDER_RELEASE
-ARG BUILDER_EPOCH
+FROM package-builder-base as package-builder-fat-base
+
+# Some extra dependencies we install early for better docker caching
+RUN ulimit -n 2000 && \
+    yum install -y autoconf bzip2 bzip2-devel findutils \
+	libuuid-devel net-tools openssl-devel pkgconfig readline-devel tar xz-devel openldap-devel
+
+##############################################################################
+# Separate build stage for luaenv to speedup builds and improve caching
+
+FROM package-builder-fat-base as luaenv-builder
+
+@IF [ ! -z "$M_all$M_lua" ]
+# These are needed for the luaenv build
+@INCLUDE ../../luaenv/Dockerfile.paths-include
+# Build Lua dist in docker, and later only package it in an RPM spec
+# This is easier to debug than build failures in an RPM build and gets cached in a layer
+RUN cd /weakforced/luaenv && ./build.sh -p /usr/share/wforce-lua-dist
+@ENDIF
+
+##############################################################################
+# Separate build stage for wforce static libs to speedup builds and caching
+
+FROM package-builder-fat-base as wforce-static-libs-builder
 
 @IF [ ! -z "$M_all$M_wforce" ]
 COPY --from=sdist /sdist /sdist
-RUN for file in /sdist/* ; do ln -s $file /root/rpmbuild/SOURCES/ ; done && ls /root/rpmbuild/SOURCES/
-@ENDIF
 
 RUN wget https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-Linux-x86_64.sh
 RUN sh cmake-3.23.2-Linux-x86_64.sh --skip-license --prefix=/usr
+
 RUN tar xvf /sdist/prometheus-cpp*Source.tar.gz
 RUN mv prometheus-cpp*Source prometheus-cpp
 RUN cd prometheus-cpp/_build && make clean && make install
 
 RUN wget https://github.com/jbeder/yaml-cpp/archive/0.8.0.tar.gz
 RUN tar xvf 0.8.0.tar.gz
-RUN cd yaml-cpp* && mkdir build && cd build && cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && make install
+RUN cd yaml-cpp* && mkdir build && cd build && cmake --install-prefix=/usr/local .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && make install
 
 RUN git clone https://github.com/open-source-parsers/jsoncpp
 RUN cd jsoncpp && git checkout tags/1.9.4 -b 1.9.4
-RUN cd jsoncpp && mkdir _build && cd _build && cmake .. -DBUILD_SHARED_LIBS=OFF && make && make install
+RUN cd jsoncpp && mkdir _build && cd _build && cmake --install-prefix=/usr/local .. -DBUILD_SHARED_LIBS=OFF && make && make install
 
 @IF [ "$BUILDER_RHEL_FLAVOUR" = "amazon" ]
 ARG DROGON_VERSION=v1.7.4
@@ -55,14 +75,46 @@ ARG DROGON_VERSION=v1.9.1
 
 RUN git clone https://github.com/drogonframework/drogon.git
 RUN cd drogon && git checkout tags/"$DROGON_VERSION" -b "$DROGON_VERSION"
-RUN cd drogon && git submodule init && git submodule update && mkdir _build && cd _build && cmake .. -DBUILD_REDIS=OFF -DBUILD_ORM=OFF -DCMAKE_BUILD_TYPE=Release && make && make install
+RUN cd drogon && git submodule init && git submodule update && mkdir _build && cd _build && cmake --install-prefix=/usr/local .. -DBUILD_REDIS=OFF -DBUILD_ORM=OFF -DCMAKE_BUILD_TYPE=Release && make && make install
 
 RUN wget https://github.com/maxmind/libmaxminddb/releases/download/1.7.1/libmaxminddb-1.7.1.tar.gz && tar xvf libmaxminddb-1.7.1.tar.gz
-RUN cd libmaxminddb-1.7.1 && ./configure --enable-shared=no && make && make install
+RUN cd libmaxminddb-1.7.1 && ./configure --prefix=/usr/local --enable-shared=no && make && make install
+@ENDIF
+
+##############################################################################
+# Our package-builder target image
+
+FROM package-builder-fat-base as package-builder
 
 ADD builder-support/specs/ /wforce/builder-support/specs/
 ADD builder/helpers /wforce/builder/helpers
 
+# Used for -p option to only build specific spec files
+ARG BUILDER_PACKAGE_MATCH
+
+ARG BUILDER_VERSION
+ARG BUILDER_RELEASE
+ARG BUILDER_EPOCH
+
+@IF [ ! -z "$M_all$M_wforce" ]
+COPY --from=sdist /sdist /sdist
+RUN for file in /sdist/* ; do ln -s $file /root/rpmbuild/SOURCES/ ; done && ls /root/rpmbuild/SOURCES/
+COPY --from=wforce-static-libs-builder /usr/local /usr/local
+@ENDIF
+
+#
+# Lua dist and modules
+#
+@IF [ ! -z "$M_all$M_lua" ]
+# This is packaged and removed in the lua-dist RPM
+COPY --from=luaenv-builder /usr/share/wforce-lua-dist /usr/share/wforce-lua-dist
+# Build and install Lua dist package
+# QA_RPATHS for: ERROR 0001: file '/usr/share/wforce-lua-dist/lib/lua/5.1/system/core.so' contains a standard rpath '/usr/lib64' in [/usr/lib64]
+RUN ulimit -n 2000 && QA_RPATHS=1 builder/helpers/build-specs.sh builder-support/specs/wforce-lua-dist.spec
+RUN ulimit -n 2000 && yum localinstall -y /root/rpmbuild/RPMS/*/wforce-lua-dist*.rpm
+@ENDIF
+
+
 @IF [ ! -z "$M_all$M_wforce" ]
 RUN yum install -y /usr/bin/python3
 RUN builder/helpers/build-specs.sh builder-support/specs/wforce.spec
@@ -72,3 +124,5 @@ RUN builder/helpers/build-specs.sh builder-support/specs/wforce.spec
 # See: https://github.com/moby/moby/issues/33733
 #RUN mv /root/rpmbuild/RPMS/* /dist/
 RUN cp -R /root/rpmbuild/RPMS/* /dist/
+
+

builder-support/dockerfiles/Dockerfile.rpmtest

@@ -10,4 +10,6 @@ COPY --from=package-builder /dist /dist
 RUN yum localinstall -y /dist/*/*.rpm
 
 # Installation tests
-@INCLUDE Dockerfile.commontest
+COPY builder-support/install-tests /wforce/builder-support/install-tests
+WORKDIR /wforce
+RUN builder-support/install-tests/test-simple-exec.sh

builder-support/dockerfiles/Dockerfile.target.centos-7

@@ -13,4 +13,6 @@ RUN echo "source /opt/rh/llvm-toolset-7.0/enable" >> /etc/bashrc
 # Do the actual rpm build
 @INCLUDE Dockerfile.rpmbuild
 
-
+# Do a test install and verify
+# Can be skipped with skiptests=1 in the environment
+@EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest

builder-support/dockerfiles/Dockerfile.target.debian-bookworm

@@ -0,0 +1,12 @@
+# First do the source builds
+@INCLUDE Dockerfile.target.sdist
+
+FROM debian:bookworm as dist-base
+ARG APT_URL
+RUN apt-get update && apt-get -y dist-upgrade
+
+@INCLUDE Dockerfile.debbuild
+
+# Do a test install and verify
+# Can be skipped with skiptests=1 in the environment
+@EXEC [ "$skiptests" = "" ] && include Dockerfile.debtest

builder-support/dockerfiles/Dockerfile.target.debian-bullseye

@@ -6,3 +6,7 @@ ARG APT_URL
 RUN apt-get update && apt-get -y dist-upgrade
 
 @INCLUDE Dockerfile.debbuild
+
+# Do a test install and verify
+# Can be skipped with skiptests=1 in the environment
+@EXEC [ "$skiptests" = "" ] && include Dockerfile.debtest

builder-support/dockerfiles/Dockerfile.target.ol-8

@@ -17,3 +17,6 @@ RUN dnf -y install dnf-plugins-core clang
 # Do the actual rpm build
 @INCLUDE Dockerfile.rpmbuild
 
+# Do a test install and verify
+# Can be skipped with skiptests=1 in the environment
+@EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest

builder-support/dockerfiles/Dockerfile.target.ol-9

@@ -16,3 +16,6 @@ RUN dnf -y install dnf-plugins-core clang
 # Do the actual rpm build
 @INCLUDE Dockerfile.rpmbuild
 
+# Do a test install and verify
+# Can be skipped with skiptests=1 in the environment
+@EXEC [ "$skiptests" = "" ] && include Dockerfile.rpmtest

builder-support/dockerfiles/Dockerfile.wforce

@@ -36,7 +36,7 @@ RUN mkdir /sdist
 RUN git clone https://github.com/jupp0r/prometheus-cpp.git
 RUN cd prometheus-cpp && git checkout tags/v1.0.1 -b v1.0.1
 RUN cd prometheus-cpp && \
-mkdir _build && cd _build && cmake .. -DBUILD_SHARED_LIBS=off -DENABLE_PULL=off \
+mkdir _build && cd _build && cmake --install-prefix=/usr/local ..  -DBUILD_SHARED_LIBS=off -DENABLE_PULL=off \
 -DENABLE_PUSH=off -DENABLE_COMPRESSION=off -DENABLE_TESTING=off -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ \
 -DCMAKE_POSITION_INDEPENDENT_CODE=ON && make && make install && make package_source
 RUN cd prometheus-cpp/_build && cp prometheus-cpp*Source.tar.gz /sdist/

builder-support/install-tests/test-simple-exec.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -ex
+
+/usr/bin/wforce --help >/dev/null
+/usr/bin/trackalert --help >/dev/null
+
+# isempty() is an extension only present in the OpenResty fork of LuaJIT,
+# the import will error in the original LuaJIT and Lua.
+/usr/bin/wforce-luajit -e 'local isempty = require "table.isempty" print(isempty({}))'
+
+# Check Lua modules are loaded
+/usr/bin/wforce-luajit -e 'require "elasticsearch" require "lualdap"'
+
+# Check that our lua-dist is the first libluajit that is found
+/sbin/ldconfig --print-cache | grep '/usr/share/wforce-lua-dist/lib/exported/libluajit'
+ldd /usr/bin/wforce | grep '/usr/share/wforce-lua-dist/lib/exported/libluajit'

builder-support/luajit-debian/compat

@@ -0,0 +1 @@
+10

builder-support/luajit-debian/control

@@ -0,0 +1,12 @@
+Source: wforce-lua-dist
+Maintainer: Neil Cook <[email protected]>
+Section: libs
+Priority: optional
+Standards-Version: 3.9.2
+Build-Depends: libldap2-dev
+
+Package: wforce-lua-dist
+Section: libs
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Description: PowerDNS Weakforced LuaJIT distribution

builder-support/luajit-debian/rules

@@ -0,0 +1,15 @@
+#!/usr/bin/make -f
+#
+# See debhelper(7) (uncomment to enable)
+# output every command that modifies files on the build system.
+#DH_VERBOSE = 1
+
+# see EXAMPLES in dpkg-buildflags(1) and read /usr/share/dpkg/*
+DPKG_EXPORT_BUILDFLAGS = 1
+include /usr/share/dpkg/default.mk
+
+%:
+	dh $@
+
+override_dh_auto_build:
+	echo '/usr/share/wforce-lua-dist/lib/exported' >wforce-lua-dist.conf

builder-support/luajit-debian/source/format

@@ -0,0 +1,2 @@
+3.0 (quilt)
+