diff --git a/configs/logging.yaml.default b/configs/logging.yaml.default index 0612def3343..d84fa5b74dc 100644 --- a/configs/logging.yaml.default +++ b/configs/logging.yaml.default @@ -11,6 +11,13 @@ logging: +# This is an example filter for the slow log, you can tweak and tune this as you like for +# your slowlog debugging. Use this with the slowlog format below, for a slowlog file. + filters: + - name: slowlog + action: accept + condition: '%<{TS_MILESTONE_SM_FINISH-TS_MILESTONE_SM_START}msdms GTE 100' + formats: # WebTrends Enhanced Log Format. # @@ -42,6 +49,9 @@ logging: - name: "extended2" format: '% - % [%] "% % %" % % % % % % % % % % % % % % %' + - name: "slowlog" + format: '[%] client_ip: %:% protocol: % url: % status: % X-ID: %<{X-Id}cqh> bytes=% client state: % server state: % tls_handshake: %<{TS_MILESTONE_TLS_HANDSHAKE_END-TS_MILESTONE_TLS_HANDSHAKE_START}msdms> ua_begin: %<{TS_MILESTONE_UA_BEGIN-TS_MILESTONE_SM_START}msdms> ua_first_read: %<{TS_MILESTONE_UA_FIRST_READ-TS_MILESTONE_SM_START}msdms> ua_read_header_done: %<{TS_MILESTONE_UA_READ_HEADER_DONE-TS_MILESTONE_SM_START}msdms> cache_open_read_begin: %<{TS_MILESTONE_CACHE_OPEN_READ_BEGIN-TS_MILESTONE_SM_START}msdms> cache_open_read_end: %<{TS_MILESTONE_CACHE_OPEN_READ_END-TS_MILESTONE_SM_START}msdms> cache_open_write_begin: %<{TS_MILESTONE_CACHE_OPEN_WRITE_BEGIN-TS_MILESTONE_SM_START}msdms> cache_open_write_end: %<{TS_MILESTONE_CACHE_OPEN_WRITE_END-TS_MILESTONE_SM_START}msdms> dns_lookup_begin: %<{TS_MILESTONE_DNS_LOOKUP_BEGIN-TS_MILESTONE_SM_START}msdms> dns_lookup_end: %<{TS_MILESTONE_DNS_LOOKUP_END-TS_MILESTONE_SM_START}msdms> server_connect: %<{TS_MILESTONE_SERVER_CONNECT-TS_MILESTONE_SM_START}msdms> server_connect_end: %<{TS_MILESTONE_SERVER_CONNECT_END-TS_MILESTONE_SM_START}msdms> server_first_read: %<{TS_MILESTONE_SERVER_FIRST_READ-TS_MILESTONE_SM_START}msdms> server_read_header_done: %<{TS_MILESTONE_SERVER_READ_HEADER_DONE-TS_MILESTONE_SM_START}msdms> server_close: %<{TS_MILESTONE_SERVER_CLOSE-TS_MILESTONE_SM_START}msdms> ua_write: %<{TS_MILESTONE_UA_BEGIN_WRITE-TS_MILESTONE_SM_START}msdms> ua_close: %<{TS_MILESTONE_UA_CLOSE-TS_MILESTONE_SM_START}msdms> sm_finish: %<{TS_MILESTONE_SM_FINISH-TS_MILESTONE_SM_START}msdms> plugin_active: %<{TS_MILESTONE_PLUGIN_ACTIVE-TS_MILESTONE_SM_START}msdms> plugin_total: %<{TS_MILESTONE_PLUGIN_TOTAL-TS_MILESTONE_SM_START}msdms>' + logs: - filename: squid format: squid diff --git a/doc/admin-guide/files/logging.yaml.en.rst b/doc/admin-guide/files/logging.yaml.en.rst index 4f46c506111..6bc7d222d36 100644 --- a/doc/admin-guide/files/logging.yaml.en.rst +++ b/doc/admin-guide/files/logging.yaml.en.rst @@ -210,6 +210,22 @@ be any one of the following: True if the value of ``field`` contains ``value`` (i.e. ``value`` is a substring of the contents of ``field``). Case-insensitive. +``LT`` + True if the value of ``field`` is less than the value of ``value``. + Only valid for integer fields. + +``LTE`` + True if the value of ``field`` is less than or equal to the value of + ``value``. Only valid for integer fields. + +``GT`` + True if the value of ``field`` is greater than the value of ``value``. + Only valid for integer fields. + +``GTE`` + True if the value of ``field`` is less than or equal to the value of + ``value``. Only valid for integer fields. + Filter Values ~~~~~~~~~~~~~ diff --git a/include/proxy/logging/LogFilter.h b/include/proxy/logging/LogFilter.h index 569d33b4d5a..a09caee17d1 100644 --- a/include/proxy/logging/LogFilter.h +++ b/include/proxy/logging/LogFilter.h @@ -66,6 +66,10 @@ class LogFilter : public RefCountObjInHeap CASE_INSENSITIVE_MATCH, CONTAIN, CASE_INSENSITIVE_CONTAIN, + LT, + LTE, + GT, + GTE, N_OPERATORS, }; @@ -92,8 +96,13 @@ class LogFilter : public RefCountObjInHeap return m_num_values; } + bool + is_wipe() const + { + return m_action == WIPE_FIELD_VALUE; + } + virtual bool toss_this_entry(LogAccess *lad) = 0; - virtual bool wipe_this_entry(LogAccess *lad) = 0; virtual void display(FILE *fd = stdout) = 0; static LogFilter *parse(const char *name, Action action, const char *condition); @@ -133,12 +142,43 @@ class LogFilterString : public LogFilter bool operator==(LogFilterString &rhs); bool toss_this_entry(LogAccess *lad) override; - bool wipe_this_entry(LogAccess *lad) override; void display(FILE *fd = stdout) override; // noncopyable LogFilterString &operator=(LogFilterString &rhs) = delete; + // This assumes that s1 is all uppercase, hence we hide this in here specifically + static const char * + strstrcase(const char *s0, const char *s1) + { + size_t len_s0 = std::strlen(s0); + size_t len_s1 = std::strlen(s1); + + if (len_s1 > len_s0) { + return nullptr; // If s1 is longer than s0, there's no match + } + + const char *end = s0 + len_s0 - len_s1; + + while (s0 < end) { + const char *h = s0; + const char *n = s1; + + while (*n && (std::toupper(static_cast(*h)) == *n)) { + ++h; + ++n; + } + + if (!*n) { + return s0; + } + + ++s0; + } + + return nullptr; + } + private: char **m_value = nullptr; // the array of values @@ -153,25 +193,21 @@ class LogFilterString : public LogFilter // (as strcmp does) using OperatorFunction = int (*)(const char *, const char *); + // return 0 if s1 is substring of s0 and 1 otherwise, to conform to strcmp etc. static int _isSubstring(const char *s0, const char *s1) { - // return 0 if s1 is substring of s0 and 1 otherwise - // this reverse behavior is to conform to the behavior of strcmp - // which returns 0 if strings match return (strstr(s0, s1) == nullptr ? 1 : 0); } - enum LengthCondition { - DATA_LENGTH_EQUAL = 0, - DATA_LENGTH_LARGER, - }; - - inline bool _checkCondition(OperatorFunction f, const char *field_value, size_t field_value_length, char **val, - LengthCondition lc); + // + static int + _isSubstringUpper(const char *s0, const char *s1) + { + return (strstrcase(s0, s1) == nullptr) ? 1 : 0; + } - inline bool _checkConditionAndWipe(OperatorFunction f, char **field_value, size_t field_value_length, char **val, - const char *uppercase_field_value, LengthCondition lc); + bool _checkConditionAndWipe(OperatorFunction f, char *field_value, size_t field_value_length, char **val, bool uppercase = false); // -- member functions that are not allowed -- LogFilterString(); @@ -193,7 +229,6 @@ class LogFilterInt : public LogFilter bool operator==(LogFilterInt &rhs); bool toss_this_entry(LogAccess *lad) override; - bool wipe_this_entry(LogAccess *lad) override; void display(FILE *fd = stdout) override; // noncopyable @@ -226,7 +261,6 @@ class LogFilterIP : public LogFilter bool operator==(LogFilterIP &rhs); bool toss_this_entry(LogAccess *lad) override; - bool wipe_this_entry(LogAccess *lad) override; void display(FILE *fd = stdout) override; // noncopyable @@ -259,7 +293,6 @@ class LogFilterList void add(LogFilter *filter, bool copy = true); bool toss_this_entry(LogAccess *lad); - bool wipe_this_entry(LogAccess *lad); LogFilter *find_by_name(const char *name); void clear(); @@ -278,18 +311,6 @@ class LogFilterList unsigned count() const; void display(FILE *fd = stdout); - bool - does_conjunction() const - { - return m_does_conjunction; - } - - void - set_conjunction(bool c) - { - m_does_conjunction = c; - } - // noncopyable // -- member functions that are not allowed -- LogFilterList(const LogFilterList &rhs) = delete; @@ -298,263 +319,5 @@ class LogFilterList private: Queue m_filter_list; - bool m_does_conjunction = true; - // If m_does_conjunction = true - // toss_this_entry returns true - // if ANY filter tosses entry away. - // If m_does_conjunction = false, - // toss this entry returns true if - // ALL filters toss away entry + bool m_has_wipe = false; }; - -/*------------------------------------------------------------------------- - Inline functions - -------------------------------------------------------------------------*/ - -/*------------------------------------------------------------------------- - _checkCondition - - check all values for a matching condition - - the arguments to the function are: - - - a function f of type OperatorFunction that determines if the - condition is true for a single filter value. Note that this function - must return 0 if the condition is true. - - the value of the field from the log record - - the length of this field - - the array of filter values to compare to note that we pass this as an - argument because it can be either m_value or m_value_uppercase - - a LengthCondition argument that determines if the length of the field value - must be equal or larger to the length of the filter value (this is to - compare strings only if really needed - ------------------------------------------------------------------------*/ - -inline bool -LogFilterString::_checkCondition(OperatorFunction f, const char *field_value, size_t field_value_length, char **val, - LengthCondition lc) -{ - bool retVal = false; - - // make single value case a little bit faster by taking it out of loop - // - if (m_num_values == 1) { - switch (lc) { - case DATA_LENGTH_EQUAL: - retVal = (field_value_length == *m_length ? ((*f)(field_value, *val) == 0 ? true : false) : false); - break; - case DATA_LENGTH_LARGER: - retVal = (field_value_length > *m_length ? ((*f)(field_value, *val) == 0 ? true : false) : false); - break; - default: - ink_assert(!"LogFilterString::checkCondition " - "unknown LengthCondition"); - } - } else { - size_t i; - switch (lc) { - case DATA_LENGTH_EQUAL: - for (i = 0; i < m_num_values; ++i) { - // condition is satisfied if f returns zero - if (field_value_length == m_length[i] && (*f)(field_value, val[i]) == 0) { - retVal = true; - break; - } - } - break; - case DATA_LENGTH_LARGER: - for (i = 0; i < m_num_values; ++i) { - // condition is satisfied if f returns zero - if (field_value_length > m_length[i] && (*f)(field_value, val[i]) == 0) { - retVal = true; - break; - } - } - break; - default: - ink_assert(!"LogFilterString::checkCondition " - "unknown LengthCondition"); - } - } - return retVal; -} - -/*--------------------------------------------------------------------------- - * find pattern from the query param - * 1) if pattern is not in the query param, return nullptr - * 2) if got the pattern in one param's value, search again until it's in one param name, or nullptr if can't find it from param -name ----------------------------------------------------------------------------*/ -static const char * -findPatternFromParamName(const char *lookup_query_param, const char *pattern) -{ - const char *pattern_in_query_param = strstr(lookup_query_param, pattern); - while (pattern_in_query_param) { - // wipe pattern in param name, need to search again if find pattern in param value - const char *param_value_str = strchr(pattern_in_query_param, '='); - if (!param_value_str) { - // no "=" after pattern_in_query_param, means pattern_in_query_param is not in the param name, and no more param after it - pattern_in_query_param = nullptr; - break; - } - const char *param_name_str = strchr(pattern_in_query_param, '&'); - if (param_name_str && param_value_str > param_name_str) { - //"=" is after "&" followd by pattern_in_query_param, means pattern_in_query_param is not in the param name - pattern_in_query_param = strstr(param_name_str, pattern); - continue; - } - // ensure pattern_in_query_param is in the param name now - break; - } - return pattern_in_query_param; -} - -/*--------------------------------------------------------------------------- - * replace param value whose name contains pattern with same count 'X' of original value str length ----------------------------------------------------------------------------*/ -static void -updatePatternForFieldValue(char **field, const char *pattern_str, int /* field_pos ATS_UNUSED */, char *buf_dest) -{ - int buf_dest_len = strlen(buf_dest); - char buf_dest_to_field[buf_dest_len + 1]; - char *temp_text = buf_dest_to_field; - memcpy(temp_text, buf_dest, (pattern_str - buf_dest)); - temp_text += (pattern_str - buf_dest); - const char *value_str = strchr(pattern_str, '='); - if (value_str) { - value_str++; - memcpy(temp_text, pattern_str, (value_str - pattern_str)); - temp_text += (value_str - pattern_str); - const char *next_param_str = strchr(value_str, '&'); - if (next_param_str) { - for (int i = 0; i < (next_param_str - value_str); i++) { - temp_text[i] = 'X'; - } - temp_text += (next_param_str - value_str); - memcpy(temp_text, next_param_str, ((buf_dest + buf_dest_len) - next_param_str)); - } else { - for (int i = 0; i < ((buf_dest + buf_dest_len) - value_str); i++) { - temp_text[i] = 'X'; - } - } - } else { - return; - } - - buf_dest_to_field[buf_dest_len] = '\0'; - strcpy(*field, buf_dest_to_field); -} - -/*--------------------------------------------------------------------------- - wipeField : Given a dest buffer, wipe the first occurrence of the value of the - field in the buffer. - ---------------------------------------------------------------------------*/ -static void -wipeField(char **field, char *pattern, const char *uppercase_field) -{ - char *buf_dest = *field; - const char *lookup_dest = uppercase_field ? uppercase_field : *field; - - if (buf_dest) { - char *query_param = strchr(buf_dest, '?'); - const char *lookup_query_param = strchr(lookup_dest, '?'); - if (!query_param || !lookup_query_param) { - return; - } - - const char *pattern_in_param_name = findPatternFromParamName(lookup_query_param, pattern); - while (pattern_in_param_name) { - int field_pos = pattern_in_param_name - lookup_query_param; - pattern_in_param_name = query_param + field_pos; - updatePatternForFieldValue(field, pattern_in_param_name, field_pos, buf_dest); - - // search new param again - const char *new_param = strchr(lookup_query_param + field_pos, '&'); - if (new_param && *(new_param + 1)) { - pattern_in_param_name = findPatternFromParamName(new_param + 1, pattern); - } else { - break; - } - } - } -} - -/*------------------------------------------------------------------------- - _checkConditionAndWipe - - check all values for a matching condition and perform wipe action - - the arguments to the function are: - - - a function f of type OperatorFunction that determines if the - condition is true for a single filter value. Note that this function - must return 0 if the condition is true. - - the value of the field from the log record - - the length of this field - - the array of filter values to compare to note that we pass this as an - argument because it can be either m_value or m_value_uppercase - - a LengthCondition argument that determines if the length of the field value - must be equal or larger to the length of the filter value (this is to - compare strings only if really needed - ------------------------------------------------------------------------*/ - -inline bool -LogFilterString::_checkConditionAndWipe(OperatorFunction f, char **field_value, size_t field_value_length, char **val, - const char *uppercase_field_value, LengthCondition lc) -{ - bool retVal = false; - - if (m_action != WIPE_FIELD_VALUE) { - return false; - } - - // make single value case a little bit faster by taking it out of loop - // - const char *lookup_field_value = uppercase_field_value ? uppercase_field_value : *field_value; - if (m_num_values == 1) { - switch (lc) { - case DATA_LENGTH_EQUAL: - retVal = (field_value_length == *m_length ? ((*f)(lookup_field_value, *val) == 0 ? true : false) : false); - if (retVal) { - wipeField(field_value, *val, uppercase_field_value); - } - break; - case DATA_LENGTH_LARGER: - retVal = (field_value_length > *m_length ? ((*f)(lookup_field_value, *val) == 0 ? true : false) : false); - if (retVal) { - wipeField(field_value, *val, uppercase_field_value); - } - break; - default: - ink_assert(!"LogFilterString::checkCondition " - "unknown LengthCondition"); - } - } else { - size_t i; - switch (lc) { - case DATA_LENGTH_EQUAL: - for (i = 0; i < m_num_values; ++i) { - // condition is satisfied if f returns zero - if (field_value_length == m_length[i] && (*f)(lookup_field_value, val[i]) == 0) { - retVal = true; - wipeField(field_value, val[i], uppercase_field_value); - } - } - break; - case DATA_LENGTH_LARGER: - for (i = 0; i < m_num_values; ++i) { - // condition is satisfied if f returns zero - if (field_value_length > m_length[i] && (*f)(lookup_field_value, val[i]) == 0) { - retVal = true; - wipeField(field_value, val[i], uppercase_field_value); - } - } - break; - default: - ink_assert(!"LogFilterString::checkConditionAndWipe " - "unknown LengthConditionAndWipe"); - } - } - return retVal; -} diff --git a/src/proxy/logging/LogField.cc b/src/proxy/logging/LogField.cc index 0373ec6dee8..f7dcbbd067a 100644 --- a/src/proxy/logging/LogField.cc +++ b/src/proxy/logging/LogField.cc @@ -367,6 +367,7 @@ LogField::LogField(const char *field, Container container) case ICFG: m_unmarshal_func = &(LogAccess::unmarshal_int_to_str); + m_type = LogField::sINT; break; case RECORD: @@ -379,6 +380,7 @@ LogField::LogField(const char *field, Container container) Note("Invalid milestone name in LogField ctor: %s", m_name); } m_unmarshal_func = &(LogAccess::unmarshal_int_to_str); + m_type = LogField::sINT; break; case MSDMS: { @@ -387,6 +389,7 @@ LogField::LogField(const char *field, Container container) Note("Invalid milestone range in LogField ctor: %s", m_name); } m_unmarshal_func = &(LogAccess::unmarshal_int_to_str); + m_type = LogField::sINT; break; } @@ -406,8 +409,8 @@ LogField::LogField(const LogField &rhs) m_agg_op(rhs.m_agg_op), m_agg_cnt(0), m_agg_val(0), - m_milestone1(TS_MILESTONE_LAST_ENTRY), - m_milestone2(TS_MILESTONE_LAST_ENTRY), + m_milestone1(rhs.m_milestone1), + m_milestone2(rhs.m_milestone2), m_time_field(rhs.m_time_field), m_alias_map(rhs.m_alias_map), m_set_func(rhs.m_set_func) @@ -514,7 +517,7 @@ LogField::updateField(LogAccess *lad, char *buf, int len) /*------------------------------------------------------------------------- LogField::marshal - This routine will marshsal the given field into the buffer provided. + This routine will marshal the given field into the buffer provided. -------------------------------------------------------------------------*/ unsigned LogField::marshal(LogAccess *lad, char *buf) diff --git a/src/proxy/logging/LogFilter.cc b/src/proxy/logging/LogFilter.cc index 8be0b3ae035..fcfe41c4e3c 100644 --- a/src/proxy/logging/LogFilter.cc +++ b/src/proxy/logging/LogFilter.cc @@ -46,8 +46,9 @@ #include "proxy/logging/LogConfig.h" #include "proxy/logging/Log.h" -const char *LogFilter::OPERATOR_NAME[] = {"MATCH", "CASE_INSENSITIVE_MATCH", "CONTAIN", "CASE_INSENSITIVE_CONTAIN"}; -const char *LogFilter::ACTION_NAME[] = {"REJECT", "ACCEPT", "WIPE_FIELD_VALUE"}; +const char *LogFilter::OPERATOR_NAME[] = { + "MATCH", "CASE_INSENSITIVE_MATCH", "CONTAIN", "CASE_INSENSITIVE_CONTAIN", "LT", "LTE", "GT", "GTE"}; +const char *LogFilter::ACTION_NAME[] = {"REJECT", "ACCEPT", "WIPE_FIELD_VALUE"}; namespace { @@ -226,6 +227,160 @@ LogFilterString::_setValues(size_t n, char **value) } } +/*--------------------------------------------------------------------------- + * find pattern from the query param + * 1) if pattern is not in the query param, return nullptr + * 2) if got the pattern in one param's value, search again until it's in one param name, or nullptr if can't find it from param +name +---------------------------------------------------------------------------*/ +static const char * +findPatternFromParamName(const char *lookup_query_param, const char *pattern, bool uppercase) +{ + const char *pattern_in_query_param = + uppercase ? LogFilterString::strstrcase(lookup_query_param, pattern) : strstr(lookup_query_param, pattern); + + while (pattern_in_query_param) { + // wipe pattern in param name, need to search again if find pattern in param value + const char *param_value_str = strchr(pattern_in_query_param, '='); + + if (!param_value_str) { + // no "=" after pattern_in_query_param, means pattern_in_query_param is not in the param name, and no more param after it + pattern_in_query_param = nullptr; + break; + } + + const char *param_name_str = strchr(pattern_in_query_param, '&'); + + if (param_name_str && param_value_str > param_name_str) { + //"=" is after "&" followd by pattern_in_query_param, means pattern_in_query_param is not in the param name + if (uppercase) { + pattern_in_query_param = LogFilterString::strstrcase(param_name_str, pattern); + } else { + pattern_in_query_param = strstr(param_name_str, pattern); + } + continue; + } + // ensure pattern_in_query_param is in the param name now + break; + } + + return pattern_in_query_param; +} + +/*--------------------------------------------------------------------------- + * replace param value whose name contains pattern with same count 'X' of original value str length +---------------------------------------------------------------------------*/ +static void +updatePatternForFieldValue(char *field, const char *pattern_str, int /* field_pos ATS_UNUSED */, char *buf_dest) +{ + int buf_dest_len = strlen(buf_dest); + char buf_dest_to_field[buf_dest_len + 1]; + char *temp_text = buf_dest_to_field; + + memcpy(temp_text, buf_dest, (pattern_str - buf_dest)); + temp_text += (pattern_str - buf_dest); + + const char *value_str = strchr(pattern_str, '='); + + if (value_str) { + value_str++; + memcpy(temp_text, pattern_str, (value_str - pattern_str)); + temp_text += (value_str - pattern_str); + + const char *next_param_str = strchr(value_str, '&'); + + if (next_param_str) { + for (int i = 0; i < (next_param_str - value_str); i++) { + temp_text[i] = 'X'; + } + temp_text += (next_param_str - value_str); + memcpy(temp_text, next_param_str, ((buf_dest + buf_dest_len) - next_param_str)); + } else { + for (int i = 0; i < ((buf_dest + buf_dest_len) - value_str); i++) { + temp_text[i] = 'X'; + } + } + } else { + return; + } + + buf_dest_to_field[buf_dest_len] = '\0'; + strcpy(field, buf_dest_to_field); +} + +/*--------------------------------------------------------------------------- + wipeField : Given a dest buffer, wipe the first occurrence of the value of the + field in the buffer. + +--------------------------------------------------------------------------*/ +static void +wipeField(char *field, char *pattern, bool uppercase) +{ + if (field) { + char *query_param = strchr(field, '?'); + + if (!query_param) { + return; + } + + const char *pattern_in_param_name = findPatternFromParamName(query_param, pattern, uppercase); + + while (pattern_in_param_name) { + int field_pos = pattern_in_param_name - query_param; + + pattern_in_param_name = query_param + field_pos; + updatePatternForFieldValue(field, pattern_in_param_name, field_pos, field); + + // search new param again + const char *new_param = strchr(query_param + field_pos, '&'); + + if (new_param && *(new_param + 1)) { + pattern_in_param_name = findPatternFromParamName(new_param + 1, pattern, uppercase); + } else { + break; + } + } + } +} + +/*------------------------------------------------------------------------- + _checkConditionAndWipe + + check all values for a matching condition and perform wipe action if + appropriate. This replaces both of the checkConditions functions. + + the arguments to the function are: + + - a function f of type OperatorFunction that determines if the + condition is true for a single filter value. Note that this function + must return 0 if the condition is true. + - the value of the field from the log record + - the length of this field + - the array of filter values to compare to note that we pass this as an + argument because it can be either m_value or m_value_uppercase + ------------------------------------------------------------------------*/ +inline bool +LogFilterString::_checkConditionAndWipe(OperatorFunction f, char *field_value, size_t field_value_length, char **val, + bool uppercase) +{ + bool retVal = false; + + for (size_t i = 0; i < m_num_values; ++i) { + // condition is satisfied if f returns zero + if (field_value_length > m_length[i] && (*f)(field_value, val[i]) == 0) { + retVal = true; + if (m_action == WIPE_FIELD_VALUE) { + Dbg(dbg_ctl_log, "entry wiped, ..."); + wipeField(field_value, val[i], uppercase); + } else { + break; // We only break here if this is not a wipe field, since we can wipe multiple values + } + } + } + + return retVal; +} + LogFilterString::LogFilterString(const char *name, LogField *field, LogFilter::Action action, LogFilter::Operator oper, char *values) : LogFilter(name, field, action, oper) @@ -306,89 +461,6 @@ LogFilterString::operator==(LogFilterString &rhs) return false; } -/*------------------------------------------------------------------------- - LogFilterString::wipe_this_entry - - For strings, we need to marshal the given string into a buffer so that we - can compare it with the filter value. Most strings are snall, so we'll - only allocate space dynamically if the marshal_len is very large (eg, - URL). - - The m_substr field tells us whether we can match based on substrings, or - whether we should compare the entire string. - -------------------------------------------------------------------------*/ - -bool -LogFilterString::wipe_this_entry(LogAccess *lad) -{ - if (m_num_values == 0 || m_field == nullptr || lad == nullptr || m_action != WIPE_FIELD_VALUE) { - return false; - } - - static const unsigned BUFSIZE = 1024; - char small_buf[BUFSIZE]; - char small_buf_upper[BUFSIZE]; - char *big_buf = nullptr; - char *big_buf_upper = nullptr; - char *buf = small_buf; - char *buf_upper = small_buf_upper; - size_t marsh_len = m_field->marshal_len(lad); // includes null termination - - if (marsh_len > BUFSIZE) { - big_buf = static_cast(ats_malloc(marsh_len)); - ink_assert(big_buf != nullptr); - buf = big_buf; - } - - ink_assert(buf != nullptr); - m_field->marshal(lad, buf); - - ink_assert(buf != nullptr); - - bool cond_satisfied = false; - switch (m_operator) { - case MATCH: - // marsh_len is an upper bound on the length of the marshalled string - // because marsh_len counts padding and the eos. So for a MATCH - // operator, we use the DATA_LENGTH_LARGER length condition rather - // than DATA_LENGTH_EQUAL, which we would use if we had the actual - // length of the string. It is probably not worth computing the - // actual length, so we just use the fact that a MATCH is not possible - // when marsh_len <= (length of the filter string) - // - cond_satisfied = _checkConditionAndWipe(&strcmp, &buf, marsh_len, m_value, nullptr, DATA_LENGTH_LARGER); - break; - case CASE_INSENSITIVE_MATCH: - cond_satisfied = _checkConditionAndWipe(&strcasecmp, &buf, marsh_len, m_value, nullptr, DATA_LENGTH_LARGER); - break; - case CONTAIN: - cond_satisfied = _checkConditionAndWipe(&_isSubstring, &buf, marsh_len, m_value, nullptr, DATA_LENGTH_LARGER); - break; - case CASE_INSENSITIVE_CONTAIN: - if (big_buf) { - big_buf_upper = static_cast(ats_malloc(static_cast(marsh_len))); - buf_upper = big_buf_upper; - } else { - buf = small_buf; // make clang happy - } - for (size_t i = 0; i < marsh_len; i++) { - buf_upper[i] = ParseRules::ink_toupper(buf[i]); - } - cond_satisfied = _checkConditionAndWipe(&_isSubstring, &buf, marsh_len, m_value_uppercase, buf_upper, DATA_LENGTH_LARGER); - break; - default: - ink_assert(!"INVALID FILTER OPERATOR"); - } - - if (cond_satisfied) { - m_field->updateField(lad, buf, strlen(buf)); - } - - ats_free(big_buf); - ats_free(big_buf_upper); - return cond_satisfied; -} - /*------------------------------------------------------------------------- LogFilterString::toss_this_entry @@ -404,65 +476,51 @@ LogFilterString::wipe_this_entry(LogAccess *lad) bool LogFilterString::toss_this_entry(LogAccess *lad) { + static const unsigned BUFSIZE = 8192; + if (m_num_values == 0 || m_field == nullptr || lad == nullptr) { return false; } - static const unsigned BUFSIZE = 1024; - char small_buf[BUFSIZE]; - char small_buf_upper[BUFSIZE]; - char *big_buf = nullptr; - char *big_buf_upper = nullptr; - char *buf = small_buf; - char *buf_upper = small_buf_upper; - size_t marsh_len = m_field->marshal_len(lad); // includes null termination + char small_buf[BUFSIZE]; + char *big_buf = nullptr; + char *buf = small_buf; + size_t marsh_len = m_field->marshal_len(lad); // includes null termination + bool cond_satisfied = false; if (marsh_len > BUFSIZE) { big_buf = static_cast(ats_malloc(static_cast(marsh_len))); ink_assert(big_buf != nullptr); buf = big_buf; } - m_field->marshal(lad, buf); - bool cond_satisfied = false; switch (m_operator) { case MATCH: // marsh_len is an upper bound on the length of the marshalled string - // because marsh_len counts padding and the eos. So for a MATCH - // operator, we use the DATA_LENGTH_LARGER length condition rather - // than DATA_LENGTH_EQUAL, which we would use if we had the actual - // length of the string. It is probably not worth computing the - // actual length, so we just use the fact that a MATCH is not possible - // when marsh_len <= (length of the filter string) - // - cond_satisfied = _checkCondition(&strcmp, buf, marsh_len, m_value, DATA_LENGTH_LARGER); + // because marsh_len counts padding and the eos. + cond_satisfied = _checkConditionAndWipe(&strcmp, buf, marsh_len, m_value); break; case CASE_INSENSITIVE_MATCH: - cond_satisfied = _checkCondition(&strcasecmp, buf, marsh_len, m_value, DATA_LENGTH_LARGER); + cond_satisfied = _checkConditionAndWipe(&strcasecmp, buf, marsh_len, m_value); break; case CONTAIN: - cond_satisfied = _checkCondition(&_isSubstring, buf, marsh_len, m_value, DATA_LENGTH_LARGER); + cond_satisfied = _checkConditionAndWipe(&_isSubstring, buf, marsh_len, m_value); break; - case CASE_INSENSITIVE_CONTAIN: { - if (big_buf) { - big_buf_upper = static_cast(ats_malloc(static_cast(marsh_len))); - buf_upper = big_buf_upper; - } else { - buf = small_buf; // make clang happy - } - for (size_t i = 0; i < marsh_len; i++) { - buf_upper[i] = ParseRules::ink_toupper(buf[i]); - } - cond_satisfied = _checkCondition(&_isSubstring, buf_upper, marsh_len, m_value_uppercase, DATA_LENGTH_LARGER); + case CASE_INSENSITIVE_CONTAIN: + // This one is special, since we use our strstrcase() function, which requires uppercase match values + cond_satisfied = _checkConditionAndWipe(&_isSubstringUpper, buf, marsh_len, m_value_uppercase, true); break; - } default: ink_assert(!"INVALID FILTER OPERATOR"); } + // Check if we should call the updateField function for this field + if (m_action == WIPE_FIELD_VALUE && cond_satisfied) { + m_field->updateField(lad, buf, strlen(buf)); + } + ats_free(big_buf); - ats_free(big_buf_upper); return ((m_action == REJECT && cond_satisfied) || (m_action == ACCEPT && !cond_satisfied)); } @@ -502,7 +560,6 @@ LogFilterInt::_setValues(size_t n, int64_t *value) } } -// TODO: ival should be int64_t int LogFilterInt::_convertStringToInt(char *value, int64_t *ival, LogFieldAliasMap *map) { @@ -557,8 +614,14 @@ LogFilterInt::LogFilterInt(const char *name, LogField *field, LogFilter::Action // pointer in @c field. if (n) { + if (m_operator >= Operator::LT && m_operator <= Operator::GTE && n > 1) { + Fatal("Operator %s can only have one value", OPERATOR_NAME[m_operator]); + return; + } + val_array = new int64_t[n]; char *t; + while (t = tok.getNext(), t != nullptr) { int64_t ival; if (!_convertStringToInt(t, &ival, field_map.get())) { @@ -624,44 +687,6 @@ LogFilterInt::operator==(LogFilterInt &rhs) return false; } -/*------------------------------------------------------------------------- - LogFilterInt::wipe_this_entry - -------------------------------------------------------------------------*/ - -bool -LogFilterInt::wipe_this_entry(LogAccess *lad) -{ - if (m_num_values == 0 || m_field == nullptr || lad == nullptr || m_action != WIPE_FIELD_VALUE) { - return false; - } - - bool cond_satisfied = false; - int64_t value; - - m_field->marshal(lad, reinterpret_cast(&value)); - // This used to do an ntohl() on value, but that breaks various filters. - // Long term we should move IPs to their own log type. - - // we don't use m_operator because we consider all operators to be - // equivalent to "MATCH" for an integer field - // - - // most common case is single value, speed it up a little bit by unrolling - // - if (m_num_values == 1) { - cond_satisfied = (value == *m_value); - } else { - for (size_t i = 0; i < m_num_values; ++i) { - if (value == m_value[i]) { - cond_satisfied = true; - break; - } - } - } - - return cond_satisfied; -} - /*------------------------------------------------------------------------- LogFilterInt::toss_this_entry -------------------------------------------------------------------------*/ @@ -677,24 +702,39 @@ LogFilterInt::toss_this_entry(LogAccess *lad) int64_t value; m_field->marshal(lad, reinterpret_cast(&value)); - // This used to do an ntohl() on value, but that breaks various filters. - // Long term we should move IPs to their own log type. - - // we don't use m_operator because we consider all operators to be - // equivalent to "MATCH" for an integer field - // - - // most common case is single value, speed it up a little bit by unrolling - // - if (m_num_values == 1) { - cond_satisfied = (value == *m_value); - } else { - for (size_t i = 0; i < m_num_values; ++i) { - if (value == m_value[i]) { - cond_satisfied = true; - break; + switch (m_operator) { + case MATCH: + // most common case is single value, speed it up a little bit by unrolling + // + if (m_num_values == 1) { + cond_satisfied = (value == *m_value); + } else { + for (size_t i = 0; i < m_num_values; ++i) { + if (value == m_value[i]) { + cond_satisfied = true; + break; + } } } + break; + case LT: + ink_assert(m_num_values == 1); + cond_satisfied = (value < *m_value); + break; + case LTE: + ink_assert(m_num_values == 1); + cond_satisfied = (value <= *m_value); + break; + case GT: + ink_assert(m_num_values == 1); + cond_satisfied = (value > *m_value); + break; + case GTE: + ink_assert(m_num_values == 1); + cond_satisfied = (value >= *m_value); + break; + default: + ink_assert(!"INVALID FILTER OPERATOR"); } return (m_action == REJECT && cond_satisfied) || (m_action == ACCEPT && !cond_satisfied); @@ -852,12 +892,6 @@ LogFilterIP::toss_this_entry(LogAccess *lad) return (m_action == REJECT && cond_satisfied) || (m_action == ACCEPT && !cond_satisfied); } -bool -LogFilterIP::wipe_this_entry(LogAccess *) -{ - return false; -} - /*------------------------------------------------------------------------- LogFilterIP::display -------------------------------------------------------------------------*/ @@ -930,24 +964,20 @@ LogFilterList::~LogFilterList() bool LogFilterList::operator==(LogFilterList &rhs) { - if (m_does_conjunction == rhs.does_conjunction()) { - LogFilter *f = first(); - LogFilter *rhsf = rhs.first(); - - while (true) { - if (!(f || rhsf)) { - return true; - } else if (!f || !rhsf) { - return false; - } else if (!filters_are_equal(f, rhsf)) { - return false; - } else { - f = next(f); - rhsf = rhs.next(rhsf); - } + LogFilter *f = first(); + LogFilter *rhsf = rhs.first(); + + while (true) { + if (!(f || rhsf)) { + return true; + } else if (!f || !rhsf) { + return false; + } else if (!filters_are_equal(f, rhsf)) { + return false; + } else { + f = next(f); + rhsf = rhs.next(rhsf); } - } else { - return false; } } @@ -984,21 +1014,8 @@ LogFilterList::add(LogFilter *filter, bool copy) } else { m_filter_list.enqueue(filter); } -} - -/*------------------------------------------------------------------------- - -------------------------------------------------------------------------*/ -bool -LogFilterList::wipe_this_entry(LogAccess *lad) -{ - bool wipeFlag = false; - for (LogFilter *f = first(); f; f = next(f)) { - if (f->wipe_this_entry(lad)) { - wipeFlag = true; - } - } - return wipeFlag; + m_has_wipe |= filter->is_wipe(); } /*------------------------------------------------------------------------- @@ -1007,25 +1024,18 @@ LogFilterList::wipe_this_entry(LogAccess *lad) bool LogFilterList::toss_this_entry(LogAccess *lad) { - if (m_does_conjunction) { - // toss if any filter rejects the entry (all filters should accept) - // - for (LogFilter *f = first(); f; f = next(f)) { - if (f->toss_this_entry(lad)) { + bool retval = false; + + for (LogFilter *f = first(); f; f = next(f)) { + if (f->toss_this_entry(lad)) { + if (m_has_wipe) { // Continue filters evaluation if there is a wipe filter in the list + retval = true; + } else { return true; } } - return false; - } else { - // toss if all filters reject the entry (any filter accepts) - // - for (LogFilter *f = first(); f; f = next(f)) { - if (!f->toss_this_entry(lad)) { - return false; - } - } - return true; } + return retval; } /*------------------------------------------------------------------------- diff --git a/src/proxy/logging/LogObject.cc b/src/proxy/logging/LogObject.cc index 1b260d49f50..d87def246c9 100644 --- a/src/proxy/logging/LogObject.cc +++ b/src/proxy/logging/LogObject.cc @@ -603,10 +603,6 @@ LogObject::log(LogAccess *lad, std::string_view text_entry) return Log::SKIP; } - if (lad && m_filter_list.wipe_this_entry(lad)) { - Dbg(dbg_ctl_log, "entry wiped, ..."); - } - if (lad && m_format->is_aggregate()) { // marshal the field data into the temp space provided by the // LogFormat object for aggregate formats