[PATCH 0 of 4] PCRE2 support

Maxim Dounin mdounin at mdounin.ru
Tue Dec 14 16:09:09 UTC 2021


Hello!

On Tue, Dec 14, 2021 at 03:09:43PM +0900, Tatsuhiko Kubo wrote:

> Hello,
> 
> I'm trying these patches and the build of nginx with the option
> "--with-pcre-jit" fails.
> 
> $ cd nginx-1.21.4 # with applied patches
> $ ./configure --with-pcre=../pcre2/pcre2-10.39 --with-pcre-jit
> $ make
> ...
> cc -c -pipe  -O -Wall -Wextra -Wpointer-arith
> -Wconditional-uninitialized -Wno-unused-parameter
> -Wno-deprecated-declarations -Werror -g  -I src/core -I src/event -I
> src/event/modules -I src/os/unix -I ../pcre2/pcre2-10.39/src/ -I objs
> \
>                 -o objs/src/core/ngx_regex.o \
>                 src/core/ngx_regex.c
> src/core/ngx_regex.c:590:15: error: use of undeclared identifier
> 'PCRE_STUDY_JIT_COMPILE'
>         opt = PCRE_STUDY_JIT_COMPILE;

Thanks, looks like a mismerge in the last patch at some point during
development.  The following patch should fix this:

diff --git a/src/core/ngx_regex.c b/src/core/ngx_regex.c
--- a/src/core/ngx_regex.c
+++ b/src/core/ngx_regex.c
@@ -585,10 +585,16 @@ ngx_regex_module_init(ngx_cycle_t *cycle
 
     rcf = (ngx_regex_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_regex_module);
 
-#if (NGX_HAVE_PCRE_JIT)
+#if (NGX_PCRE2 || NGX_HAVE_PCRE_JIT)
+
     if (rcf->pcre_jit) {
+#if (NGX_PCRE2)
+        opt = 1;
+#else
         opt = PCRE_STUDY_JIT_COMPILE;
+#endif
     }
+
 #endif
 
     ngx_regex_malloc_init(cycle->pool);


Full series updated:

# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1639495851 -10800
#      Tue Dec 14 18:30:51 2021 +0300
# Node ID 4f979a9f2a68b25b8b2ce8a0bd15671095f6c327
# Parent  a7a77549265ef46f1f0fdb3897f4beabf9e09c40
Core: fixed ngx_pcre_studies cleanup.

If a configuration parsing fails for some reason, ngx_regex_module_init()
is not called, and ngx_pcre_studies remained set despite the fact that
the pool it was allocated from is already freed.  This might result in
a segmentation fault during runtime regular expression compilation, such
as in SSI, for example, in the single process mode, or if a worker process
dies and respawn from a master process in such an inconsistent state.

Fix is to clear ngx_pcre_studies from the pool cleanup handler (which is
anyway used to free JIT-compiled patterns).

diff --git a/src/core/ngx_regex.c b/src/core/ngx_regex.c
--- a/src/core/ngx_regex.c
+++ b/src/core/ngx_regex.c
@@ -10,15 +10,14 @@
 
 
 typedef struct {
-    ngx_flag_t  pcre_jit;
+    ngx_flag_t   pcre_jit;
+    ngx_list_t  *studies;
 } ngx_regex_conf_t;
 
 
 static void * ngx_libc_cdecl ngx_regex_malloc(size_t size);
 static void ngx_libc_cdecl ngx_regex_free(void *p);
-#if (NGX_HAVE_PCRE_JIT)
-static void ngx_pcre_free_studies(void *data);
-#endif
+static void ngx_regex_cleanup(void *data);
 
 static ngx_int_t ngx_regex_module_init(ngx_cycle_t *cycle);
 
@@ -248,18 +247,17 @@ ngx_regex_free(void *p)
 }
 
 
+static void
+ngx_regex_cleanup(void *data)
+{
 #if (NGX_HAVE_PCRE_JIT)
-
-static void
-ngx_pcre_free_studies(void *data)
-{
-    ngx_list_t *studies = data;
+    ngx_regex_conf_t *rcf = data;
 
     ngx_uint_t        i;
     ngx_list_part_t  *part;
     ngx_regex_elt_t  *elts;
 
-    part = &studies->part;
+    part = &rcf->studies->part;
     elts = part->elts;
 
     for (i = 0; /* void */ ; i++) {
@@ -274,56 +272,50 @@ ngx_pcre_free_studies(void *data)
             i = 0;
         }
 
-        if (elts[i].regex->extra != NULL) {
-            pcre_free_study(elts[i].regex->extra);
-        }
-    }
-}
-
-#endif
-
-
-static ngx_int_t
-ngx_regex_module_init(ngx_cycle_t *cycle)
-{
-    int               opt;
-    const char       *errstr;
-    ngx_uint_t        i;
-    ngx_list_part_t  *part;
-    ngx_regex_elt_t  *elts;
-
-    opt = 0;
-
-#if (NGX_HAVE_PCRE_JIT)
-    {
-    ngx_regex_conf_t    *rcf;
-    ngx_pool_cleanup_t  *cln;
-
-    rcf = (ngx_regex_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_regex_module);
-
-    if (rcf->pcre_jit) {
-        opt = PCRE_STUDY_JIT_COMPILE;
-
         /*
          * The PCRE JIT compiler uses mmap for its executable codes, so we
          * have to explicitly call the pcre_free_study() function to free
          * this memory.
          */
 
-        cln = ngx_pool_cleanup_add(cycle->pool, 0);
-        if (cln == NULL) {
-            return NGX_ERROR;
+        if (elts[i].regex->extra != NULL) {
+            pcre_free_study(elts[i].regex->extra);
         }
+    }
+#endif
 
-        cln->handler = ngx_pcre_free_studies;
-        cln->data = ngx_pcre_studies;
-    }
+    /*
+     * On configuration parsing errors ngx_regex_module_init() will not
+     * be called.  Make sure ngx_pcre_studies is properly cleared anyway.
+     */
+
+    ngx_pcre_studies = NULL;
+}
+
+
+static ngx_int_t
+ngx_regex_module_init(ngx_cycle_t *cycle)
+{
+    int                opt;
+    const char        *errstr;
+    ngx_uint_t         i;
+    ngx_list_part_t   *part;
+    ngx_regex_elt_t   *elts;
+    ngx_regex_conf_t  *rcf;
+
+    opt = 0;
+
+    rcf = (ngx_regex_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_regex_module);
+
+#if (NGX_HAVE_PCRE_JIT)
+    if (rcf->pcre_jit) {
+        opt = PCRE_STUDY_JIT_COMPILE;
     }
 #endif
 
     ngx_regex_malloc_init(cycle->pool);
 
-    part = &ngx_pcre_studies->part;
+    part = &rcf->studies->part;
     elts = part->elts;
 
     for (i = 0; /* void */ ; i++) {
@@ -374,7 +366,8 @@ ngx_regex_module_init(ngx_cycle_t *cycle
 static void *
 ngx_regex_create_conf(ngx_cycle_t *cycle)
 {
-    ngx_regex_conf_t  *rcf;
+    ngx_regex_conf_t    *rcf;
+    ngx_pool_cleanup_t  *cln;
 
     rcf = ngx_pcalloc(cycle->pool, sizeof(ngx_regex_conf_t));
     if (rcf == NULL) {
@@ -383,11 +376,21 @@ ngx_regex_create_conf(ngx_cycle_t *cycle
 
     rcf->pcre_jit = NGX_CONF_UNSET;
 
-    ngx_pcre_studies = ngx_list_create(cycle->pool, 8, sizeof(ngx_regex_elt_t));
-    if (ngx_pcre_studies == NULL) {
+    cln = ngx_pool_cleanup_add(cycle->pool, 0);
+    if (cln == NULL) {
         return NULL;
     }
 
+    cln->handler = ngx_regex_cleanup;
+    cln->data = rcf;
+
+    rcf->studies = ngx_list_create(cycle->pool, 8, sizeof(ngx_regex_elt_t));
+    if (rcf->studies == NULL) {
+        return NULL;
+    }
+
+    ngx_pcre_studies = rcf->studies;
+
     return rcf;
 }
 
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1639497274 -10800
#      Tue Dec 14 18:54:34 2021 +0300
# Node ID 8f00e80ade356c768a678c44711e8c2a7223ceef
# Parent  4f979a9f2a68b25b8b2ce8a0bd15671095f6c327
Core: ngx_regex.c style cleanup.

Notably, ngx_pcre_pool and ngx_pcre_studies are renamed to ngx_regex_pool
and ngx_regex_studies, respectively.

diff --git a/src/core/ngx_regex.c b/src/core/ngx_regex.c
--- a/src/core/ngx_regex.c
+++ b/src/core/ngx_regex.c
@@ -64,8 +64,8 @@ ngx_module_t  ngx_regex_module = {
 };
 
 
-static ngx_pool_t  *ngx_pcre_pool;
-static ngx_list_t  *ngx_pcre_studies;
+static ngx_pool_t  *ngx_regex_pool;
+static ngx_list_t  *ngx_regex_studies;
 
 
 void
@@ -79,14 +79,14 @@ ngx_regex_init(void)
 static ngx_inline void
 ngx_regex_malloc_init(ngx_pool_t *pool)
 {
-    ngx_pcre_pool = pool;
+    ngx_regex_pool = pool;
 }
 
 
 static ngx_inline void
 ngx_regex_malloc_done(void)
 {
-    ngx_pcre_pool = NULL;
+    ngx_regex_pool = NULL;
 }
 
 
@@ -112,13 +112,13 @@ ngx_regex_compile(ngx_regex_compile_t *r
            rc->err.len = ngx_snprintf(rc->err.data, rc->err.len,
                               "pcre_compile() failed: %s in \"%V\"",
                                errstr, &rc->pattern)
-                      - rc->err.data;
+                         - rc->err.data;
 
         } else {
            rc->err.len = ngx_snprintf(rc->err.data, rc->err.len,
                               "pcre_compile() failed: %s in \"%V\" at \"%s\"",
                                errstr, &rc->pattern, rc->pattern.data + erroff)
-                      - rc->err.data;
+                         - rc->err.data;
         }
 
         return NGX_ERROR;
@@ -133,8 +133,8 @@ ngx_regex_compile(ngx_regex_compile_t *r
 
     /* do not study at runtime */
 
-    if (ngx_pcre_studies != NULL) {
-        elt = ngx_list_push(ngx_pcre_studies);
+    if (ngx_regex_studies != NULL) {
+        elt = ngx_list_push(ngx_regex_studies);
         if (elt == NULL) {
             goto nomem;
         }
@@ -229,11 +229,8 @@ ngx_regex_exec_array(ngx_array_t *a, ngx
 static void * ngx_libc_cdecl
 ngx_regex_malloc(size_t size)
 {
-    ngx_pool_t      *pool;
-    pool = ngx_pcre_pool;
-
-    if (pool) {
-        return ngx_palloc(pool, size);
+    if (ngx_regex_pool) {
+        return ngx_palloc(ngx_regex_pool, size);
     }
 
     return NULL;
@@ -286,10 +283,10 @@ ngx_regex_cleanup(void *data)
 
     /*
      * On configuration parsing errors ngx_regex_module_init() will not
-     * be called.  Make sure ngx_pcre_studies is properly cleared anyway.
+     * be called.  Make sure ngx_regex_studies is properly cleared anyway.
      */
 
-    ngx_pcre_studies = NULL;
+    ngx_regex_studies = NULL;
 }
 
 
@@ -357,7 +354,7 @@ ngx_regex_module_init(ngx_cycle_t *cycle
 
     ngx_regex_malloc_done();
 
-    ngx_pcre_studies = NULL;
+    ngx_regex_studies = NULL;
 
     return NGX_OK;
 }
@@ -389,7 +386,7 @@ ngx_regex_create_conf(ngx_cycle_t *cycle
         return NULL;
     }
 
-    ngx_pcre_studies = rcf->studies;
+    ngx_regex_studies = rcf->studies;
 
     return rcf;
 }
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1639497275 -10800
#      Tue Dec 14 18:54:35 2021 +0300
# Node ID 0a11d3d92a5d4c1ddcfdd1f07423d59b7c2aa863
# Parent  8f00e80ade356c768a678c44711e8c2a7223ceef
Configure: simplified PCRE compilation.

Removed ICC-specific PCRE optimizations which tried to link with PCRE
object files instead of the library.  Made compiler-specific code
minimal.

diff --git a/auto/lib/pcre/conf b/auto/lib/pcre/conf
--- a/auto/lib/pcre/conf
+++ b/auto/lib/pcre/conf
@@ -4,81 +4,24 @@
 
 
 if [ $PCRE != NONE ]; then
+
+    have=NGX_PCRE . auto/have
+
+    if [ "$NGX_PLATFORM" = win32 ]; then
+        have=PCRE_STATIC . auto/have
+    fi
+
     CORE_INCS="$CORE_INCS $PCRE"
+    CORE_DEPS="$CORE_DEPS $PCRE/pcre.h"
 
     case "$NGX_CC_NAME" in
 
         msvc | owc | bcc)
-            have=NGX_PCRE . auto/have
-            have=PCRE_STATIC . auto/have
-            CORE_DEPS="$CORE_DEPS $PCRE/pcre.h"
             LINK_DEPS="$LINK_DEPS $PCRE/pcre.lib"
             CORE_LIBS="$CORE_LIBS $PCRE/pcre.lib"
         ;;
 
-        icc)
-            have=NGX_PCRE . auto/have
-            CORE_DEPS="$CORE_DEPS $PCRE/pcre.h"
-
-            LINK_DEPS="$LINK_DEPS $PCRE/.libs/libpcre.a"
-
-            echo $ngx_n "checking for PCRE library ...$ngx_c"
-
-            if [ -f $PCRE/pcre.h ]; then
-                ngx_pcre_ver=`grep PCRE_MAJOR $PCRE/pcre.h \
-                              | sed -e 's/^.*PCRE_MAJOR.* \(.*\)$/\1/'`
-
-            else if [ -f $PCRE/configure.in ]; then
-                ngx_pcre_ver=`grep PCRE_MAJOR= $PCRE/configure.in \
-                              | sed -e 's/^.*=\(.*\)$/\1/'`
-
-            else
-                ngx_pcre_ver=`grep pcre_major, $PCRE/configure.ac \
-                              | sed -e 's/^.*pcre_major,.*\[\(.*\)\].*$/\1/'`
-            fi
-            fi
-
-            echo " $ngx_pcre_ver major version found"
-
-            # to allow -ipo optimization we link with the *.o but not library
-
-            case "$ngx_pcre_ver" in
-                4|5)
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre.o"
-                ;;
-
-                6)
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_chartables.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_compile.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_exec.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_fullinfo.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_globals.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_tables.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_try_flipped.o"
-                ;;
-
-                *)
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_chartables.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_compile.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_exec.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_fullinfo.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_globals.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_tables.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_try_flipped.o"
-                    CORE_LIBS="$CORE_LIBS $PCRE/pcre_newline.o"
-                ;;
-
-            esac
-        ;;
-
         *)
-            have=NGX_PCRE . auto/have
-
-            if [ "$NGX_PLATFORM" = win32 ]; then
-                have=PCRE_STATIC . auto/have
-            fi
-
-            CORE_DEPS="$CORE_DEPS $PCRE/pcre.h"
             LINK_DEPS="$LINK_DEPS $PCRE/.libs/libpcre.a"
             CORE_LIBS="$CORE_LIBS $PCRE/.libs/libpcre.a"
         ;;
# HG changeset patch
# User Maxim Dounin <mdounin at mdounin.ru>
# Date 1639497637 -10800
#      Tue Dec 14 19:00:37 2021 +0300
# Node ID 6915b92704ceca0030ea6d9e19f37e868a8d2303
# Parent  0a11d3d92a5d4c1ddcfdd1f07423d59b7c2aa863
PCRE2 library support.

The PCRE2 library is now used by default if found, instead of the
original PCRE library.  If needed for some reason, this can be disabled
with the --without-pcre2 configure option.

To make it possible to specify paths to the library and include files
via --with-cc-opt / --with-ld-opt, the library is first tested without
any additional paths and options.  If this fails, the pcre2-config script
is used.

Similarly to the original PCRE library, it is now possible to build PCRE2
from sources with nginx configure, by using the --with-pcre= option.
It automatically detects if PCRE or PCRE2 sources are provided.

Note that compiling PCRE2 10.33 and later requires inttypes.h.  When
compiling on Windows with MSVC, inttypes.h is only available starting
with MSVC 2013.  In older versions some replacement needs to be provided
("echo '#include <stdint.h>' > pcre2-10.xx/src/inttypes.h" is good enough
for MSVC 2010).

The interface on nginx side remains unchanged.

diff --git a/auto/lib/pcre/conf b/auto/lib/pcre/conf
--- a/auto/lib/pcre/conf
+++ b/auto/lib/pcre/conf
@@ -5,29 +5,61 @@
 
 if [ $PCRE != NONE ]; then
 
-    have=NGX_PCRE . auto/have
+    if [ -f $PCRE/src/pcre2.h.generic ]; then
+
+        PCRE_LIBRARY=PCRE2
+
+        have=NGX_PCRE . auto/have
+        have=NGX_PCRE2 . auto/have
+
+        if [ "$NGX_PLATFORM" = win32 ]; then
+            have=PCRE2_STATIC . auto/have
+        fi
+
+        CORE_INCS="$CORE_INCS $PCRE/src/"
+        CORE_DEPS="$CORE_DEPS $PCRE/src/pcre2.h"
 
-    if [ "$NGX_PLATFORM" = win32 ]; then
-        have=PCRE_STATIC . auto/have
-    fi
+        case "$NGX_CC_NAME" in
+
+            msvc)
+                LINK_DEPS="$LINK_DEPS $PCRE/src/pcre2-8.lib"
+                CORE_LIBS="$CORE_LIBS $PCRE/src/pcre2-8.lib"
+            ;;
 
-    CORE_INCS="$CORE_INCS $PCRE"
-    CORE_DEPS="$CORE_DEPS $PCRE/pcre.h"
+            *)
+                LINK_DEPS="$LINK_DEPS $PCRE/.libs/libpcre2-8.a"
+                CORE_LIBS="$CORE_LIBS $PCRE/.libs/libpcre2-8.a"
+            ;;
 
-    case "$NGX_CC_NAME" in
+        esac
 
-        msvc | owc | bcc)
-            LINK_DEPS="$LINK_DEPS $PCRE/pcre.lib"
-            CORE_LIBS="$CORE_LIBS $PCRE/pcre.lib"
-        ;;
+    else
+
+        PCRE_LIBRARY=PCRE
+
+        have=NGX_PCRE . auto/have
+
+        if [ "$NGX_PLATFORM" = win32 ]; then
+            have=PCRE_STATIC . auto/have
+        fi
+
+        CORE_INCS="$CORE_INCS $PCRE"
+        CORE_DEPS="$CORE_DEPS $PCRE/pcre.h"
 
-        *)
-            LINK_DEPS="$LINK_DEPS $PCRE/.libs/libpcre.a"
-            CORE_LIBS="$CORE_LIBS $PCRE/.libs/libpcre.a"
-        ;;
+        case "$NGX_CC_NAME" in
+
+            msvc | owc | bcc)
+                LINK_DEPS="$LINK_DEPS $PCRE/pcre.lib"
+                CORE_LIBS="$CORE_LIBS $PCRE/pcre.lib"
+            ;;
 
-    esac
+            *)
+                LINK_DEPS="$LINK_DEPS $PCRE/.libs/libpcre.a"
+                CORE_LIBS="$CORE_LIBS $PCRE/.libs/libpcre.a"
+            ;;
 
+        esac
+    fi
 
     if [ $PCRE_JIT = YES ]; then
         have=NGX_HAVE_PCRE_JIT . auto/have
@@ -37,8 +69,48 @@ if [ $PCRE != NONE ]; then
 else
 
     if [ "$NGX_PLATFORM" != win32 ]; then
+        PCRE=NO
+    fi
 
-        PCRE=NO
+    if [ $PCRE = NO -a $PCRE2 != DISABLED ]; then
+
+        ngx_feature="PCRE2 library"
+        ngx_feature_name="NGX_PCRE2"
+        ngx_feature_run=no
+        ngx_feature_incs="#define PCRE2_CODE_UNIT_WIDTH 8
+                          #include <pcre2.h>"
+        ngx_feature_path=
+        ngx_feature_libs="-lpcre2-8"
+        ngx_feature_test="pcre2_code *re;
+                          re = pcre2_compile(NULL, 0, 0, NULL, NULL, NULL);
+                          if (re == NULL) return 1"
+        . auto/feature
+
+        if [ $ngx_found = no ]; then
+
+            # pcre2-config
+
+            ngx_pcre2_prefix=`pcre2-config --prefix 2>/dev/null`
+
+            if [ -n "$ngx_pcre2_prefix" ]; then
+                ngx_feature="PCRE2 library in $ngx_pcre2_prefix"
+                ngx_feature_path=`pcre2-config --cflags \
+                                  | sed -n -e 's/.*-I *\([^ ][^ ]*\).*/\1/p'`
+                ngx_feature_libs=`pcre2-config --libs8`
+                . auto/feature
+            fi
+        fi
+
+        if [ $ngx_found = yes ]; then
+            have=NGX_PCRE . auto/have
+            CORE_INCS="$CORE_INCS $ngx_feature_path"
+            CORE_LIBS="$CORE_LIBS $ngx_feature_libs"
+            PCRE=YES
+            PCRE_LIBRARY=PCRE2
+        fi
+    fi
+
+    if [ $PCRE = NO ]; then
 
         ngx_feature="PCRE library"
         ngx_feature_name="NGX_PCRE"
@@ -114,6 +186,7 @@ else
             CORE_INCS="$CORE_INCS $ngx_feature_path"
             CORE_LIBS="$CORE_LIBS $ngx_feature_libs"
             PCRE=YES
+            PCRE_LIBRARY=PCRE
         fi
 
         if [ $PCRE = YES ]; then
diff --git a/auto/lib/pcre/make b/auto/lib/pcre/make
--- a/auto/lib/pcre/make
+++ b/auto/lib/pcre/make
@@ -3,36 +3,138 @@
 # Copyright (C) Nginx, Inc.
 
 
-case "$NGX_CC_NAME" in
+if [ $PCRE_LIBRARY = PCRE2 ]; then
+
+    # PCRE2
+
+    if [ $NGX_CC_NAME = msvc ]; then
+
+        # With PCRE2, it is not possible to compile all sources.
+        # Since list of source files changes between versions, we
+        # test files which might not be present.
 
-    msvc)
-        ngx_makefile=makefile.msvc
-        ngx_opt="CPU_OPT=\"$CPU_OPT\" LIBC=$LIBC"
-        ngx_pcre="PCRE=\"$PCRE\""
-    ;;
+        ngx_pcre_srcs="pcre2_auto_possess.c \
+                       pcre2_chartables.c \
+                       pcre2_compile.c \
+                       pcre2_config.c \
+                       pcre2_context.c \
+                       pcre2_dfa_match.c \
+                       pcre2_error.c \
+                       pcre2_jit_compile.c \
+                       pcre2_maketables.c \
+                       pcre2_match.c \
+                       pcre2_match_data.c \
+                       pcre2_newline.c \
+                       pcre2_ord2utf.c \
+                       pcre2_pattern_info.c \
+                       pcre2_string_utils.c \
+                       pcre2_study.c \
+                       pcre2_substitute.c \
+                       pcre2_substring.c \
+                       pcre2_tables.c \
+                       pcre2_ucd.c \
+                       pcre2_valid_utf.c \
+                       pcre2_xclass.c"
+
+        ngx_pcre_test="pcre2_convert.c \
+                       pcre2_extuni.c \
+                       pcre2_find_bracket.c \
+                       pcre2_script_run.c \
+                       pcre2_serialize.c"
+
+        for ngx_src in $ngx_pcre_test
+        do
+            if [ -f $PCRE/src/$ngx_src ]; then
+                ngx_pcre_srcs="$ngx_pcre_srcs $ngx_src"
+            fi
+        done
 
-    owc)
-        ngx_makefile=makefile.owc
-        ngx_opt="CPU_OPT=\"$CPU_OPT\""
-        ngx_pcre=`echo PCRE=\"$PCRE\" | sed -e "s/\//$ngx_regex_dirsep/g"`
-    ;;
+        ngx_pcre_objs=`echo $ngx_pcre_srcs \
+            | sed -e "s#\([^ ]*\.\)c#\1$ngx_objext#g"`
+
+        ngx_pcre_srcs=`echo $ngx_pcre_srcs \
+            | sed -e "s/  *\([^ ][^ ]*\)/$ngx_regex_cont\1/g"`
+        ngx_pcre_objs=`echo $ngx_pcre_objs \
+            | sed -e "s/  *\([^ ][^ ]*\)/$ngx_regex_cont\1/g"`
+
+        cat << END                                            >> $NGX_MAKEFILE
+
+PCRE_CFLAGS =	-O2 -Ob1 -Oi -Gs $LIBC $CPU_OPT
+PCRE_FLAGS =	-DHAVE_CONFIG_H -DPCRE2_STATIC -DPCRE2_CODE_UNIT_WIDTH=8 \\
+		-DHAVE_MEMMOVE
+
+PCRE_SRCS =	 $ngx_pcre_srcs
+PCRE_OBJS =	 $ngx_pcre_objs
+
+$PCRE/src/pcre2.h:
+	cd $PCRE/src \\
+	&& copy /y config.h.generic config.h \\
+	&& copy /y pcre2.h.generic pcre2.h \\
+	&& copy /y pcre2_chartables.c.dist pcre2_chartables.c
 
-    bcc)
-        ngx_makefile=makefile.bcc
-        ngx_opt="-DCPU_OPT=\"$CPU_OPT\""
-        ngx_pcre=`echo \-DPCRE=\"$PCRE\" | sed -e "s/\//$ngx_regex_dirsep/g"`
-    ;;
+$PCRE/src/pcre2-8.lib:	$PCRE/src/pcre2.h $NGX_MAKEFILE
+	cd $PCRE/src \\
+	&& cl -nologo -c \$(PCRE_CFLAGS) -I . \$(PCRE_FLAGS) \$(PCRE_SRCS) \\
+	&& link -lib -out:pcre2-8.lib -verbose:lib \$(PCRE_OBJS)
+
+END
+
+    else
+
+        cat << END                                            >> $NGX_MAKEFILE
+
+$PCRE/src/pcre2.h:	$PCRE/Makefile
 
-    *)
-        ngx_makefile=
-    ;;
+$PCRE/Makefile:	$NGX_MAKEFILE
+	cd $PCRE \\
+	&& if [ -f Makefile ]; then \$(MAKE) distclean; fi \\
+	&& CC="\$(CC)" CFLAGS="$PCRE_OPT" \\
+	./configure --disable-shared $PCRE_CONF_OPT
 
-esac
+$PCRE/.libs/libpcre2-8.a:	$PCRE/Makefile
+	cd $PCRE \\
+	&& \$(MAKE) libpcre2-8.la
+
+END
+
+    fi
 
 
-if [ -n "$ngx_makefile" ]; then
+else
+
+    # PCRE
+
+    case "$NGX_CC_NAME" in
+
+        msvc)
+            ngx_makefile=makefile.msvc
+            ngx_opt="CPU_OPT=\"$CPU_OPT\" LIBC=$LIBC"
+            ngx_pcre="PCRE=\"$PCRE\""
+        ;;
+
+        owc)
+            ngx_makefile=makefile.owc
+            ngx_opt="CPU_OPT=\"$CPU_OPT\""
+            ngx_pcre=`echo PCRE=\"$PCRE\" | sed -e "s/\//$ngx_regex_dirsep/g"`
+        ;;
 
-    cat << END                                                >> $NGX_MAKEFILE
+        bcc)
+            ngx_makefile=makefile.bcc
+            ngx_opt="-DCPU_OPT=\"$CPU_OPT\""
+            ngx_pcre=`echo \-DPCRE=\"$PCRE\" \
+                | sed -e "s/\//$ngx_regex_dirsep/g"`
+        ;;
+
+        *)
+            ngx_makefile=
+        ;;
+
+    esac
+
+
+    if [ -n "$ngx_makefile" ]; then
+
+        cat << END                                            >> $NGX_MAKEFILE
 
 `echo "$PCRE/pcre.lib:	$PCRE/pcre.h $NGX_MAKEFILE"			\
 	| sed -e "s/\//$ngx_regex_dirsep/g"`
@@ -43,9 +145,9 @@ if [ -n "$ngx_makefile" ]; then
 
 END
 
-else
+    else
 
-    cat << END                                                >> $NGX_MAKEFILE
+        cat << END                                            >> $NGX_MAKEFILE
 
 $PCRE/pcre.h:	$PCRE/Makefile
 
@@ -61,4 +163,6 @@ else
 
 END
 
+    fi
+
 fi
diff --git a/auto/options b/auto/options
--- a/auto/options
+++ b/auto/options
@@ -146,6 +146,7 @@ PCRE=NONE
 PCRE_OPT=
 PCRE_CONF_OPT=
 PCRE_JIT=NO
+PCRE2=YES
 
 USE_OPENSSL=NO
 OPENSSL=NONE
@@ -357,6 +358,7 @@ use the \"--with-mail_ssl_module\" optio
         --with-pcre=*)                   PCRE="$value"              ;;
         --with-pcre-opt=*)               PCRE_OPT="$value"          ;;
         --with-pcre-jit)                 PCRE_JIT=YES               ;;
+        --without-pcre2)                 PCRE2=DISABLED             ;;
 
         --with-openssl=*)                OPENSSL="$value"           ;;
         --with-openssl-opt=*)            OPENSSL_OPT="$value"       ;;
@@ -573,6 +575,7 @@ cat << END
   --with-pcre=DIR                    set path to PCRE library sources
   --with-pcre-opt=OPTIONS            set additional build options for PCRE
   --with-pcre-jit                    build PCRE with JIT compilation support
+  --without-pcre2                    do not use PCRE2 library
 
   --with-zlib=DIR                    set path to zlib library sources
   --with-zlib-opt=OPTIONS            set additional build options for zlib
diff --git a/auto/summary b/auto/summary
--- a/auto/summary
+++ b/auto/summary
@@ -16,9 +16,9 @@ if [ $USE_PCRE = DISABLED ]; then
 
 else
     case $PCRE in
-        YES)   echo "  + using system PCRE library" ;;
+        YES)   echo "  + using system $PCRE_LIBRARY library" ;;
         NONE)  echo "  + PCRE library is not used" ;;
-        *)     echo "  + using PCRE library: $PCRE" ;;
+        *)     echo "  + using $PCRE_LIBRARY library: $PCRE" ;;
     esac
 fi
 
diff --git a/src/core/ngx_regex.c b/src/core/ngx_regex.c
--- a/src/core/ngx_regex.c
+++ b/src/core/ngx_regex.c
@@ -15,8 +15,16 @@ typedef struct {
 } ngx_regex_conf_t;
 
 
+static ngx_inline void ngx_regex_malloc_init(ngx_pool_t *pool);
+static ngx_inline void ngx_regex_malloc_done(void);
+
+#if (NGX_PCRE2)
+static void * ngx_libc_cdecl ngx_regex_malloc(size_t size, void *data);
+static void ngx_libc_cdecl ngx_regex_free(void *p, void *data);
+#else
 static void * ngx_libc_cdecl ngx_regex_malloc(size_t size);
 static void ngx_libc_cdecl ngx_regex_free(void *p);
+#endif
 static void ngx_regex_cleanup(void *data);
 
 static ngx_int_t ngx_regex_module_init(ngx_cycle_t *cycle);
@@ -64,15 +72,24 @@ ngx_module_t  ngx_regex_module = {
 };
 
 
-static ngx_pool_t  *ngx_regex_pool;
-static ngx_list_t  *ngx_regex_studies;
+static ngx_pool_t             *ngx_regex_pool;
+static ngx_list_t             *ngx_regex_studies;
+static ngx_uint_t              ngx_regex_direct_alloc;
+
+#if (NGX_PCRE2)
+static pcre2_compile_context  *ngx_regex_compile_context;
+static pcre2_match_data       *ngx_regex_match_data;
+static ngx_uint_t              ngx_regex_match_data_size;
+#endif
 
 
 void
 ngx_regex_init(void)
 {
+#if !(NGX_PCRE2)
     pcre_malloc = ngx_regex_malloc;
     pcre_free = ngx_regex_free;
+#endif
 }
 
 
@@ -80,6 +97,7 @@ static ngx_inline void
 ngx_regex_malloc_init(ngx_pool_t *pool)
 {
     ngx_regex_pool = pool;
+    ngx_regex_direct_alloc = (pool == NULL) ? 1 : 0;
 }
 
 
@@ -87,9 +105,146 @@ static ngx_inline void
 ngx_regex_malloc_done(void)
 {
     ngx_regex_pool = NULL;
+    ngx_regex_direct_alloc = 0;
 }
 
 
+#if (NGX_PCRE2)
+
+ngx_int_t
+ngx_regex_compile(ngx_regex_compile_t *rc)
+{
+    int                     n, errcode;
+    char                   *p;
+    u_char                  errstr[128];
+    size_t                  erroff;
+    pcre2_code             *re;
+    ngx_regex_elt_t        *elt;
+    pcre2_general_context  *gctx;
+    pcre2_compile_context  *cctx;
+
+    if (ngx_regex_compile_context == NULL) {
+        /*
+         * Allocte a compile context if not yet allocated.  This uses
+         * direct allocations from heap, so the result can be cached
+         * even at runtime.
+         */
+
+        ngx_regex_malloc_init(NULL);
+
+        gctx = pcre2_general_context_create(ngx_regex_malloc, ngx_regex_free,
+                                            NULL);
+        if (gctx == NULL) {
+            ngx_regex_malloc_done();
+            goto nomem;
+        }
+
+        cctx = pcre2_compile_context_create(gctx);
+        if (cctx == NULL) {
+            pcre2_general_context_free(gctx);
+            ngx_regex_malloc_done();
+            goto nomem;
+        }
+
+        ngx_regex_compile_context = cctx;
+
+        pcre2_general_context_free(gctx);
+        ngx_regex_malloc_done();
+    }
+
+    ngx_regex_malloc_init(rc->pool);
+
+    re = pcre2_compile(rc->pattern.data, rc->pattern.len,
+                       (uint32_t) rc->options, &errcode, &erroff,
+                       ngx_regex_compile_context);
+
+    /* ensure that there is no current pool */
+    ngx_regex_malloc_done();
+
+    if (re == NULL) {
+        pcre2_get_error_message(errcode, errstr, 128);
+
+        if ((size_t) erroff == rc->pattern.len) {
+            rc->err.len = ngx_snprintf(rc->err.data, rc->err.len,
+                              "pcre2_compile() failed: %s in \"%V\"",
+                               errstr, &rc->pattern)
+                          - rc->err.data;
+
+        } else {
+            rc->err.len = ngx_snprintf(rc->err.data, rc->err.len,
+                              "pcre2_compile() failed: %s in \"%V\" at \"%s\"",
+                               errstr, &rc->pattern, rc->pattern.data + erroff)
+                          - rc->err.data;
+        }
+
+        return NGX_ERROR;
+    }
+
+    rc->regex = re;
+
+    /* do not study at runtime */
+
+    if (ngx_regex_studies != NULL) {
+        elt = ngx_list_push(ngx_regex_studies);
+        if (elt == NULL) {
+            goto nomem;
+        }
+
+        elt->regex = rc->regex;
+        elt->name = rc->pattern.data;
+    }
+
+    n = pcre2_pattern_info(re, PCRE2_INFO_CAPTURECOUNT, &rc->captures);
+    if (n < 0) {
+        p = "pcre2_pattern_info(\"%V\", PCRE2_INFO_CAPTURECOUNT) failed: %d";
+        goto failed;
+    }
+
+    if (rc->captures == 0) {
+        return NGX_OK;
+    }
+
+    n = pcre2_pattern_info(re, PCRE2_INFO_NAMECOUNT, &rc->named_captures);
+    if (n < 0) {
+        p = "pcre2_pattern_info(\"%V\", PCRE2_INFO_NAMECOUNT) failed: %d";
+        goto failed;
+    }
+
+    if (rc->named_captures == 0) {
+        return NGX_OK;
+    }
+
+    n = pcre2_pattern_info(re, PCRE2_INFO_NAMEENTRYSIZE, &rc->name_size);
+    if (n < 0) {
+        p = "pcre2_pattern_info(\"%V\", PCRE2_INFO_NAMEENTRYSIZE) failed: %d";
+        goto failed;
+    }
+
+    n = pcre2_pattern_info(re, PCRE2_INFO_NAMETABLE, &rc->names);
+    if (n < 0) {
+        p = "pcre2_pattern_info(\"%V\", PCRE2_INFO_NAMETABLE) failed: %d";
+        goto failed;
+    }
+
+    return NGX_OK;
+
+failed:
+
+    rc->err.len = ngx_snprintf(rc->err.data, rc->err.len, p, &rc->pattern, n)
+                  - rc->err.data;
+    return NGX_ERROR;
+
+nomem:
+
+    rc->err.len = ngx_snprintf(rc->err.data, rc->err.len,
+                               "regex \"%V\" compilation failed: no memory",
+                               &rc->pattern)
+                  - rc->err.data;
+    return NGX_ERROR;
+}
+
+#else
+
 ngx_int_t
 ngx_regex_compile(ngx_regex_compile_t *rc)
 {
@@ -192,6 +347,74 @@ nomem:
     return NGX_ERROR;
 }
 
+#endif
+
+
+#if (NGX_PCRE2)
+
+ngx_int_t
+ngx_regex_exec(ngx_regex_t *re, ngx_str_t *s, int *captures, ngx_uint_t size)
+{
+    size_t      *ov;
+    ngx_int_t    rc;
+    ngx_uint_t   n, i;
+
+    /*
+     * The pcre2_match() function might allocate memory for backtracking
+     * frames, typical allocations are from 40k and above.  So the allocator
+     * is configured to do direct allocations from heap during matching.
+     */
+
+    ngx_regex_malloc_init(NULL);
+
+    if (ngx_regex_match_data == NULL
+        || size > ngx_regex_match_data_size)
+    {
+        /*
+         * Allocate a match data if not yet allocated or smaller than
+         * needed.
+         */
+
+        if (ngx_regex_match_data) {
+            pcre2_match_data_free(ngx_regex_match_data);
+        }
+
+        ngx_regex_match_data_size = size;
+        ngx_regex_match_data = pcre2_match_data_create(size / 3, NULL);
+
+        if (ngx_regex_match_data == NULL) {
+            rc = PCRE2_ERROR_NOMEMORY;
+            goto failed;
+        }
+    }
+
+    rc = pcre2_match(re, s->data, s->len, 0, 0, ngx_regex_match_data, NULL);
+
+    if (rc < 0) {
+        goto failed;
+    }
+
+    n = pcre2_get_ovector_count(ngx_regex_match_data);
+    ov = pcre2_get_ovector_pointer(ngx_regex_match_data);
+
+    if (n > size / 3) {
+        n = size / 3;
+    }
+
+    for (i = 0; i < n; i++) {
+        captures[i * 2] = ov[i * 2];
+        captures[i * 2 + 1] = ov[i * 2 + 1];
+    }
+
+failed:
+
+    ngx_regex_malloc_done();
+
+    return rc;
+}
+
+#endif
+
 
 ngx_int_t
 ngx_regex_exec_array(ngx_array_t *a, ngx_str_t *s, ngx_log_t *log)
@@ -226,6 +449,35 @@ ngx_regex_exec_array(ngx_array_t *a, ngx
 }
 
 
+#if (NGX_PCRE2)
+
+static void * ngx_libc_cdecl
+ngx_regex_malloc(size_t size, void *data)
+{
+    if (ngx_regex_pool) {
+        return ngx_palloc(ngx_regex_pool, size);
+    }
+
+    if (ngx_regex_direct_alloc) {
+        return ngx_alloc(size, ngx_cycle->log);
+    }
+
+    return NULL;
+}
+
+
+static void ngx_libc_cdecl
+ngx_regex_free(void *p, void *data)
+{
+    if (ngx_regex_direct_alloc) {
+        ngx_free(p);
+    }
+
+    return;
+}
+
+#else
+
 static void * ngx_libc_cdecl
 ngx_regex_malloc(size_t size)
 {
@@ -243,11 +495,13 @@ ngx_regex_free(void *p)
     return;
 }
 
+#endif
+
 
 static void
 ngx_regex_cleanup(void *data)
 {
-#if (NGX_HAVE_PCRE_JIT)
+#if (NGX_PCRE2 || NGX_HAVE_PCRE_JIT)
     ngx_regex_conf_t *rcf = data;
 
     ngx_uint_t        i;
@@ -272,12 +526,17 @@ ngx_regex_cleanup(void *data)
         /*
          * The PCRE JIT compiler uses mmap for its executable codes, so we
          * have to explicitly call the pcre_free_study() function to free
-         * this memory.
+         * this memory.  In PCRE2, we call the pcre2_code_free() function
+         * for the same reason.
          */
 
+#if (NGX_PCRE2)
+        pcre2_code_free(elts[i].regex);
+#else
         if (elts[i].regex->extra != NULL) {
             pcre_free_study(elts[i].regex->extra);
         }
+#endif
     }
 #endif
 
@@ -287,6 +546,26 @@ ngx_regex_cleanup(void *data)
      */
 
     ngx_regex_studies = NULL;
+
+#if (NGX_PCRE2)
+
+    /*
+     * Free compile context and match data.  If needed at runtime by
+     * the new cycle, these will be re-allocated.
+     */
+
+    if (ngx_regex_compile_context) {
+        pcre2_compile_context_free(ngx_regex_compile_context);
+        ngx_regex_compile_context = NULL;
+    }
+
+    if (ngx_regex_match_data) {
+        pcre2_match_data_free(ngx_regex_match_data);
+        ngx_regex_match_data = NULL;
+        ngx_regex_match_data_size = 0;
+    }
+
+#endif
 }
 
 
@@ -294,7 +573,9 @@ static ngx_int_t
 ngx_regex_module_init(ngx_cycle_t *cycle)
 {
     int                opt;
+#if !(NGX_PCRE2)
     const char        *errstr;
+#endif
     ngx_uint_t         i;
     ngx_list_part_t   *part;
     ngx_regex_elt_t   *elts;
@@ -304,10 +585,16 @@ ngx_regex_module_init(ngx_cycle_t *cycle
 
     rcf = (ngx_regex_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_regex_module);
 
-#if (NGX_HAVE_PCRE_JIT)
+#if (NGX_PCRE2 || NGX_HAVE_PCRE_JIT)
+
     if (rcf->pcre_jit) {
+#if (NGX_PCRE2)
+        opt = 1;
+#else
         opt = PCRE_STUDY_JIT_COMPILE;
+#endif
     }
+
 #endif
 
     ngx_regex_malloc_init(cycle->pool);
@@ -327,6 +614,23 @@ ngx_regex_module_init(ngx_cycle_t *cycle
             i = 0;
         }
 
+#if (NGX_PCRE2)
+
+        if (opt) {
+            int  n;
+
+            n = pcre2_jit_compile(elts[i].regex, PCRE2_JIT_COMPLETE);
+
+            if (n != 0) {
+                ngx_log_error(NGX_LOG_INFO, cycle->log, 0,
+                              "pcre2_jit_compile() failed: %d in \"%s\", "
+                              "ignored",
+                              n, elts[i].name);
+            }
+        }
+
+#else
+
         elts[i].regex->extra = pcre_study(elts[i].regex->code, opt, &errstr);
 
         if (errstr != NULL) {
@@ -350,11 +654,15 @@ ngx_regex_module_init(ngx_cycle_t *cycle
             }
         }
 #endif
+#endif
     }
 
     ngx_regex_malloc_done();
 
     ngx_regex_studies = NULL;
+#if (NGX_PCRE2)
+    ngx_regex_compile_context = NULL;
+#endif
 
     return NGX_OK;
 }
@@ -412,7 +720,21 @@ ngx_regex_pcre_jit(ngx_conf_t *cf, void 
         return NGX_CONF_OK;
     }
 
-#if (NGX_HAVE_PCRE_JIT)
+#if (NGX_PCRE2)
+    {
+    int       r;
+    uint32_t  jit;
+
+    jit = 0;
+    r = pcre2_config(PCRE2_CONFIG_JIT, &jit);
+
+    if (r != 0 || jit != 1) {
+        ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+                           "PCRE2 library does not support JIT");
+        *fp = 0;
+    }
+    }
+#elif (NGX_HAVE_PCRE_JIT)
     {
     int  jit, r;
 
diff --git a/src/core/ngx_regex.h b/src/core/ngx_regex.h
--- a/src/core/ngx_regex.h
+++ b/src/core/ngx_regex.h
@@ -12,19 +12,31 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 
+
+#if (NGX_PCRE2)
+
+#define PCRE2_CODE_UNIT_WIDTH  8
+#include <pcre2.h>
+
+#define NGX_REGEX_NO_MATCHED   PCRE2_ERROR_NOMATCH   /* -1 */
+#define NGX_REGEX_CASELESS     PCRE2_CASELESS
+
+typedef pcre2_code  ngx_regex_t;
+
+#else
+
 #include <pcre.h>
 
-
-#define NGX_REGEX_NO_MATCHED  PCRE_ERROR_NOMATCH   /* -1 */
-
-#define NGX_REGEX_CASELESS    PCRE_CASELESS
-
+#define NGX_REGEX_NO_MATCHED   PCRE_ERROR_NOMATCH   /* -1 */
+#define NGX_REGEX_CASELESS     PCRE_CASELESS
 
 typedef struct {
     pcre        *code;
     pcre_extra  *extra;
 } ngx_regex_t;
 
+#endif
+
 
 typedef struct {
     ngx_str_t     pattern;
@@ -49,10 +61,20 @@ typedef struct {
 void ngx_regex_init(void);
 ngx_int_t ngx_regex_compile(ngx_regex_compile_t *rc);
 
+#if (NGX_PCRE2)
+
+ngx_int_t ngx_regex_exec(ngx_regex_t *re, ngx_str_t *s, int *captures,
+    ngx_uint_t size);
+#define ngx_regex_exec_n       "pcre2_match()"
+
+#else
+
 #define ngx_regex_exec(re, s, captures, size)                                \
     pcre_exec(re->code, re->extra, (const char *) (s)->data, (s)->len, 0, 0, \
               captures, size)
-#define ngx_regex_exec_n      "pcre_exec()"
+#define ngx_regex_exec_n       "pcre_exec()"
+
+#endif
 
 ngx_int_t ngx_regex_exec_array(ngx_array_t *a, ngx_str_t *s, ngx_log_t *log);
 

-- 
Maxim Dounin
http://mdounin.ru/


More information about the nginx-devel mailing list