Skip to content

Commit 7729ade

Browse files
xiangwang1xiangwang1
committed
Merge branch develop to master
2 parents 90cd186 + aea440a commit 7729ade

File tree

10 files changed

+156
-15
lines changed

10 files changed

+156
-15
lines changed

‎CHANGELOG.md

+8
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,14 @@
22

33
This is a list of notable changes to Hyperscan, in reverse chronological order.
44

5+
## [5.1.1] 2019-04-03
6+
- Add extra detection and handling when invalid rose programs are triggered.
7+
- Bugfix for issue #136: fix CMake parsing of CPU architecure for GCC-9.
8+
- Bugfix for issue #137: avoid file path impact on fat runtime build.
9+
- Bugfix for issue #141: fix rose literal programs for multi-pattern
10+
matching when no pattern ids are provided.
11+
- Bugfix for issue #144: fix library install path in pkg-config files.
12+
513
## [5.1.0] 2019-01-17
614
- Improve DFA state compression by wide-state optimization to reduce bytecode
715
size.

‎CMakeLists.txt

+3-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ project (hyperscan C CXX)
33

44
set (HS_MAJOR_VERSION 5)
55
set (HS_MINOR_VERSION 1)
6-
set (HS_PATCH_VERSION 0)
6+
set (HS_PATCH_VERSION 1)
77
set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION})
88

99
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
@@ -191,6 +191,8 @@ else()
191191
set (EXEC_ARGS ${CC_ARG1} -c -Q --help=target -march=native -mtune=native)
192192
execute_process(COMMAND ${CMAKE_C_COMPILER} ${EXEC_ARGS}
193193
OUTPUT_VARIABLE _GCC_OUTPUT)
194+
string(FIND "${_GCC_OUTPUT}" "Known" POS)
195+
string(SUBSTRING "${_GCC_OUTPUT}" 0 ${POS} _GCC_OUTPUT)
194196
string(REGEX REPLACE ".*march=[ \t]*([^ \n]*)[ \n].*" "\\1"
195197
GNUCC_ARCH "${_GCC_OUTPUT}")
196198

‎chimera/libch.pc.in

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
prefix=@CMAKE_INSTALL_PREFIX@
22
exec_prefix=@CMAKE_INSTALL_PREFIX@
3-
libdir=@CMAKE_INSTALL_PREFIX@/lib
4-
includedir=@CMAKE_INSTALL_PREFIX@/include
3+
libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@
4+
includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@
55

66
Name: libch
77
Description: Intel(R) Chimera Library

‎cmake/build_wrapper.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ PREFIX=$1
99
KEEPSYMS_IN=$2
1010
shift 2
1111
# $@ contains the actual build command
12-
OUT=$(echo "$@" | sed 's/.* -o \(.*\.o\).*/\1/')
12+
OUT=$(echo "$@" | rev | cut -d ' ' -f 2- | rev | sed 's/.* -o \(.*\.o\).*/\1/')
1313
trap cleanup INT QUIT EXIT
1414
SYMSFILE=$(mktemp -p /tmp ${PREFIX}_rename.syms.XXXXX)
1515
KEEPSYMS=$(mktemp -p /tmp keep.syms.XXXXX)

‎libhs.pc.in

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
prefix=@CMAKE_INSTALL_PREFIX@
22
exec_prefix=@CMAKE_INSTALL_PREFIX@
3-
libdir=@CMAKE_INSTALL_PREFIX@/lib
4-
includedir=@CMAKE_INSTALL_PREFIX@/include
3+
libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@
4+
includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@
55

66
Name: libhs
77
Description: Intel(R) Hyperscan Library

‎src/hs_common.h

+11-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2015-2017, Intel Corporation
2+
* Copyright (c) 2015-2019, Intel Corporation
33
*
44
* Redistribution and use in source and binary forms, with or without
55
* modification, are permitted provided that the following conditions are met:
@@ -577,6 +577,16 @@ hs_error_t HS_CDECL hs_valid_platform(void);
577577
*/
578578
#define HS_INSUFFICIENT_SPACE (-12)
579579

580+
/**
581+
* Unexpected internal error.
582+
*
583+
* This error indicates that there was unexpected matching behaviors. This
584+
* could be related to invalid usage of stream and scratch space or invalid memory
585+
* operations by users.
586+
*
587+
*/
588+
#define HS_UNKNOWN_ERROR (-13)
589+
580590
/** @} */
581591

582592
#ifdef __cplusplus

‎src/rose/program_runtime.c

+33
Original file line numberDiff line numberDiff line change
@@ -2771,6 +2771,12 @@ hwlmcb_rv_t roseRunProgram(const struct RoseEngine *t,
27712771
work_done = 1;
27722772
}
27732773
PROGRAM_NEXT_INSTRUCTION
2774+
2775+
default: {
2776+
assert(0); // unreachable
2777+
scratch->core_info.status |= STATUS_ERROR;
2778+
return HWLM_TERMINATE_MATCHING;
2779+
}
27742780
}
27752781
}
27762782

@@ -2807,6 +2813,10 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t,
28072813
const char *pc_base = getByOffset(t, programOffset);
28082814
const char *pc = pc_base;
28092815

2816+
// If this program has an effect, work_done will be set to one (which may
2817+
// allow the program to squash groups).
2818+
int work_done = 0;
2819+
28102820
struct RoseContext *tctxt = &scratch->tctxt;
28112821

28122822
assert(*(const u8 *)pc != ROSE_INSTR_END);
@@ -2887,6 +2897,7 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t,
28872897
INVALID_EKEY) == HWLM_TERMINATE_MATCHING) {
28882898
return HWLM_TERMINATE_MATCHING;
28892899
}
2900+
work_done = 1;
28902901
}
28912902
L_PROGRAM_NEXT_INSTRUCTION
28922903

@@ -2896,6 +2907,7 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t,
28962907
ri->ekey) == HWLM_TERMINATE_MATCHING) {
28972908
return HWLM_TERMINATE_MATCHING;
28982909
}
2910+
work_done = 1;
28992911
}
29002912
L_PROGRAM_NEXT_INSTRUCTION
29012913

@@ -2906,6 +2918,7 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t,
29062918
INVALID_EKEY) == HWLM_TERMINATE_MATCHING) {
29072919
return HWLM_TERMINATE_MATCHING;
29082920
}
2921+
work_done = 1;
29092922
}
29102923
L_PROGRAM_NEXT_INSTRUCTION
29112924

@@ -2933,6 +2946,7 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t,
29332946
ekey) == HWLM_TERMINATE_MATCHING) {
29342947
return HWLM_TERMINATE_MATCHING;
29352948
}
2949+
work_done = 1;
29362950
}
29372951
L_PROGRAM_NEXT_INSTRUCTION
29382952

@@ -2963,6 +2977,16 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t,
29632977
}
29642978
L_PROGRAM_NEXT_INSTRUCTION
29652979

2980+
L_PROGRAM_CASE(SQUASH_GROUPS) {
2981+
assert(popcount64(ri->groups) == 63); // Squash only one group.
2982+
if (work_done) {
2983+
tctxt->groups &= ri->groups;
2984+
DEBUG_PRINTF("squash groups 0x%llx -> 0x%llx\n", ri->groups,
2985+
tctxt->groups);
2986+
}
2987+
}
2988+
L_PROGRAM_NEXT_INSTRUCTION
2989+
29662990
L_PROGRAM_CASE(CHECK_LONG_LIT) {
29672991
const char nocase = 0;
29682992
if (!roseCheckLongLiteral(t, scratch, end, ri->lit_offset,
@@ -3011,6 +3035,12 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t,
30113035
}
30123036
L_PROGRAM_NEXT_INSTRUCTION
30133037

3038+
L_PROGRAM_CASE(CLEAR_WORK_DONE) {
3039+
DEBUG_PRINTF("clear work_done flag\n");
3040+
work_done = 0;
3041+
}
3042+
L_PROGRAM_NEXT_INSTRUCTION
3043+
30143044
L_PROGRAM_CASE(SET_LOGICAL) {
30153045
DEBUG_PRINTF("set logical value of lkey %u, offset_adjust=%d\n",
30163046
ri->lkey, ri->offset_adjust);
@@ -3048,11 +3078,14 @@ hwlmcb_rv_t roseRunProgram_l(const struct RoseEngine *t,
30483078
== HWLM_TERMINATE_MATCHING) {
30493079
return HWLM_TERMINATE_MATCHING;
30503080
}
3081+
work_done = 1;
30513082
}
30523083
L_PROGRAM_NEXT_INSTRUCTION
30533084

30543085
default: {
30553086
assert(0); // unreachable
3087+
scratch->core_info.status |= STATUS_ERROR;
3088+
return HWLM_TERMINATE_MATCHING;
30563089
}
30573090
}
30583091
}

‎src/runtime.c

+52-7
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ void populateCoreInfo(struct hs_scratch *s, const struct RoseEngine *rose,
151151
}
152152

153153
#define STATUS_VALID_BITS \
154-
(STATUS_TERMINATED | STATUS_EXHAUSTED | STATUS_DELAY_DIRTY)
154+
(STATUS_TERMINATED | STATUS_EXHAUSTED | STATUS_DELAY_DIRTY | STATUS_ERROR)
155155

156156
/** \brief Retrieve status bitmask from stream state. */
157157
static really_inline
@@ -428,7 +428,10 @@ hs_error_t HS_CDECL hs_scan(const hs_database_t *db, const char *data,
428428
}
429429

430430
done_scan:
431-
if (told_to_stop_matching(scratch)) {
431+
if (unlikely(internal_matching_error(scratch))) {
432+
unmarkScratchInUse(scratch);
433+
return HS_UNKNOWN_ERROR;
434+
} else if (told_to_stop_matching(scratch)) {
432435
unmarkScratchInUse(scratch);
433436
return HS_SCAN_TERMINATED;
434437
}
@@ -447,8 +450,17 @@ hs_error_t HS_CDECL hs_scan(const hs_database_t *db, const char *data,
447450
}
448451

449452
set_retval:
453+
if (unlikely(internal_matching_error(scratch))) {
454+
unmarkScratchInUse(scratch);
455+
return HS_UNKNOWN_ERROR;
456+
}
457+
450458
if (rose->flushCombProgramOffset) {
451459
if (roseRunFlushCombProgram(rose, scratch, ~0ULL) == MO_HALT_MATCHING) {
460+
if (unlikely(internal_matching_error(scratch))) {
461+
unmarkScratchInUse(scratch);
462+
return HS_UNKNOWN_ERROR;
463+
}
452464
unmarkScratchInUse(scratch);
453465
return HS_SCAN_TERMINATED;
454466
}
@@ -626,7 +638,7 @@ void report_eod_matches(hs_stream_t *id, hs_scratch_t *scratch,
626638
char *state = getMultiState(id);
627639
u8 status = getStreamStatus(state);
628640

629-
if (status & (STATUS_TERMINATED | STATUS_EXHAUSTED)) {
641+
if (status & (STATUS_TERMINATED | STATUS_EXHAUSTED | STATUS_ERROR)) {
630642
DEBUG_PRINTF("stream is broken, just freeing storage\n");
631643
return;
632644
}
@@ -748,6 +760,10 @@ hs_error_t HS_CDECL hs_reset_and_copy_stream(hs_stream_t *to_id,
748760
return HS_SCRATCH_IN_USE;
749761
}
750762
report_eod_matches(to_id, scratch, onEvent, context);
763+
if (unlikely(internal_matching_error(scratch))) {
764+
unmarkScratchInUse(scratch);
765+
return HS_UNKNOWN_ERROR;
766+
}
751767
unmarkScratchInUse(scratch);
752768
}
753769

@@ -863,9 +879,11 @@ hs_error_t hs_scan_stream_internal(hs_stream_t *id, const char *data,
863879
char *state = getMultiState(id);
864880

865881
u8 status = getStreamStatus(state);
866-
if (status & (STATUS_TERMINATED | STATUS_EXHAUSTED)) {
882+
if (status & (STATUS_TERMINATED | STATUS_EXHAUSTED | STATUS_ERROR)) {
867883
DEBUG_PRINTF("stream is broken, halting scan\n");
868-
if (status & STATUS_TERMINATED) {
884+
if (status & STATUS_ERROR) {
885+
return HS_UNKNOWN_ERROR;
886+
} else if (status & STATUS_TERMINATED) {
869887
return HS_SCAN_TERMINATED;
870888
} else {
871889
return HS_SUCCESS;
@@ -937,7 +955,9 @@ hs_error_t hs_scan_stream_internal(hs_stream_t *id, const char *data,
937955

938956
setStreamStatus(state, scratch->core_info.status);
939957

940-
if (likely(!can_stop_matching(scratch))) {
958+
if (unlikely(internal_matching_error(scratch))) {
959+
return HS_UNKNOWN_ERROR;
960+
} else if (likely(!can_stop_matching(scratch))) {
941961
maintainHistoryBuffer(rose, state, data, length);
942962
id->offset += length; /* maintain offset */
943963

@@ -986,13 +1006,22 @@ hs_error_t HS_CDECL hs_close_stream(hs_stream_t *id, hs_scratch_t *scratch,
9861006
return HS_SCRATCH_IN_USE;
9871007
}
9881008
report_eod_matches(id, scratch, onEvent, context);
1009+
if (unlikely(internal_matching_error(scratch))) {
1010+
unmarkScratchInUse(scratch);
1011+
return HS_UNKNOWN_ERROR;
1012+
}
9891013
unmarkScratchInUse(scratch);
9901014
}
9911015

9921016
if (id->rose->flushCombProgramOffset && !told_to_stop_matching(scratch)) {
9931017
if (roseRunFlushCombProgram(id->rose, scratch, ~0ULL)
9941018
== MO_HALT_MATCHING) {
9951019
scratch->core_info.status |= STATUS_TERMINATED;
1020+
if (unlikely(internal_matching_error(scratch))) {
1021+
unmarkScratchInUse(scratch);
1022+
return HS_UNKNOWN_ERROR;
1023+
}
1024+
unmarkScratchInUse(scratch);
9961025
}
9971026
}
9981027

@@ -1018,13 +1047,22 @@ hs_error_t HS_CDECL hs_reset_stream(hs_stream_t *id, UNUSED unsigned int flags,
10181047
return HS_SCRATCH_IN_USE;
10191048
}
10201049
report_eod_matches(id, scratch, onEvent, context);
1050+
if (unlikely(internal_matching_error(scratch))) {
1051+
unmarkScratchInUse(scratch);
1052+
return HS_UNKNOWN_ERROR;
1053+
}
10211054
unmarkScratchInUse(scratch);
10221055
}
10231056

10241057
if (id->rose->flushCombProgramOffset && !told_to_stop_matching(scratch)) {
10251058
if (roseRunFlushCombProgram(id->rose, scratch, ~0ULL)
10261059
== MO_HALT_MATCHING) {
10271060
scratch->core_info.status |= STATUS_TERMINATED;
1061+
if (unlikely(internal_matching_error(scratch))) {
1062+
unmarkScratchInUse(scratch);
1063+
return HS_UNKNOWN_ERROR;
1064+
}
1065+
unmarkScratchInUse(scratch);
10281066
}
10291067
}
10301068

@@ -1139,7 +1177,10 @@ hs_error_t HS_CDECL hs_scan_vector(const hs_database_t *db,
11391177
if (onEvent) {
11401178
report_eod_matches(id, scratch, onEvent, context);
11411179

1142-
if (told_to_stop_matching(scratch)) {
1180+
if (unlikely(internal_matching_error(scratch))) {
1181+
unmarkScratchInUse(scratch);
1182+
return HS_UNKNOWN_ERROR;
1183+
} else if (told_to_stop_matching(scratch)) {
11431184
unmarkScratchInUse(scratch);
11441185
return HS_SCAN_TERMINATED;
11451186
}
@@ -1237,6 +1278,10 @@ hs_error_t HS_CDECL hs_reset_and_expand_stream(hs_stream_t *to_stream,
12371278
return HS_SCRATCH_IN_USE;
12381279
}
12391280
report_eod_matches(to_stream, scratch, onEvent, context);
1281+
if (unlikely(internal_matching_error(scratch))) {
1282+
unmarkScratchInUse(scratch);
1283+
return HS_UNKNOWN_ERROR;
1284+
}
12401285
unmarkScratchInUse(scratch);
12411286
}
12421287

‎src/scratch.h

+10-1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ struct catchup_pq {
8484
* history. */
8585
#define STATUS_DELAY_DIRTY (1U << 2)
8686

87+
/** \brief Status flag: Unexpected Rose program error. */
88+
#define STATUS_ERROR (1U << 3)
89+
8790
/** \brief Core information about the current scan, used everywhere. */
8891
struct core_info {
8992
void *userContext; /**< user-supplied context */
@@ -229,7 +232,13 @@ char told_to_stop_matching(const struct hs_scratch *scratch) {
229232

230233
static really_inline
231234
char can_stop_matching(const struct hs_scratch *scratch) {
232-
return scratch->core_info.status & (STATUS_TERMINATED | STATUS_EXHAUSTED);
235+
return scratch->core_info.status &
236+
(STATUS_TERMINATED | STATUS_EXHAUSTED | STATUS_ERROR);
237+
}
238+
239+
static really_inline
240+
char internal_matching_error(const struct hs_scratch *scratch) {
241+
return scratch->core_info.status & STATUS_ERROR;
233242
}
234243

235244
/**

0 commit comments

Comments
 (0)