From a4baccb2758fb738613fdcb580e6676187a273cd Mon Sep 17 00:00:00 2001 From: Chris Park Date: Sun, 8 Jan 2023 14:36:35 -0800 Subject: [PATCH] updates --- diskover-web/CHANGELOG.md | 11 +++++ diskover-web/public/js/diskover.js | 67 +++++--------------------- diskover-web/public/nav.php | 2 +- diskover-web/public/search.php | 23 +++++++-- diskover-web/public/selectindices.php | 2 +- diskover-web/src/diskover/Diskover.php | 11 +++++ diskover-web/src/diskover/version.php | 2 +- diskover/CHANGELOG.md | 9 ++++ diskover/diskover.py | 32 ++++++++++-- diskover/diskover_elasticsearch.py | 6 +++ diskover/diskover_helpers.py | 20 ++++++++ 11 files changed, 118 insertions(+), 67 deletions(-) diff --git a/diskover-web/CHANGELOG.md b/diskover-web/CHANGELOG.md index 618de32e..445f354b 100644 --- a/diskover-web/CHANGELOG.md +++ b/diskover-web/CHANGELOG.md @@ -1,5 +1,16 @@ # Diskover-web v2 Community Edition Change Log +# [2.0.7] - 2023-01-08 +### fixed +- Cross-Site Scripting (XSS) vulnerability in nav.php +- php warning when reloading indices and there is a corrupt index +- delete index not working on indices page +### added +- show multi-fields on help page fields section and filter fields, e.g. field.subfield +- user alert when trying to sort on a field not found in index or trying to sort on a text field +- index name to delete prompt when deleting index on indices page + + # [2.0.6] - 2022-11-06 ### fixed - issue searching for full paths to hidden dot files/folders and files with double extensions (e.g. tar.gz) diff --git a/diskover-web/public/js/diskover.js b/diskover-web/public/js/diskover.js index 8318f1fe..4736b840 100644 --- a/diskover-web/public/js/diskover.js +++ b/diskover-web/public/js/diskover.js @@ -757,18 +757,17 @@ function clipboardNotice() { // delete index check function checkIndexDel() { - var indices = document.getElementsByClassName('indexcheck'); - var checked = 0; + var indices = document.getElementsByName('delindices_arr[]'); + var checked = document.getElementsByName('delindices_arr[]').length; + var indices_names = []; Array.from(indices).forEach(function(item, index){ - if($(item).prop("checked") == true){ - checked += 1; - } + indices_names.push(item.value); }); if (checked == 0) { alert("select at least one index") return false; } - if (confirm('Are you sure you want to remove the selected indices?')) { + if (confirm('Are you sure you want to remove the selected ' + checked + ' indices? (' + indices_names.join(", ") + ')')) { // submit form $('#form-deleteindex').submit(); } else { @@ -785,36 +784,6 @@ function checkForceIndexDel(i) { } } -// index alias check -function checkIndexAlias() { - var indices = document.getElementsByClassName('indexcheck'); - var checked = 0; - Array.from(indices).forEach(function(item, index){ - if($(item).prop("checked") == true){ - checked += 1; - } - }); - if (checked == 0) { - alert("select at least one index") - return false; - } - if (!$('#aliasname').val()) { - alert("no alias name") - return false; - } - if ($('#aliasname').val().substring(0, 9) !== "diskover-" || $('#aliasname').val().length < 10) { - console.log($('#aliasname').val().substring(0, 9)) - alert("alias name needs to start with diskover-") - return false; - } - if (confirm('Are you sure you want to change the selected indices alias?')) { - // submit form - return true; - } else { - return false; - } -} - // select index check function checkSelectedIndex() { var indices = document.getElementsByClassName('indexcheck'); @@ -833,26 +802,16 @@ function checkSelectedIndex() { } function addHidden() { - var item = document.getElementsByClassName('indexcheck'); - if($(item).prop("checked") == true){ - var id = 'hidden_index_del_' + item.value; - $('#form-deleteindex').append(''); - } else { - var id = 'hidden_index_del_' + item.value; - $('#' + id).remove(); - } -} - -function toggleHiddenInput(item) { - if($(item).prop("checked") == true){ - var id = 'hidden_index_del_' + item.value; - if (!$('#' + id).length) { + var indices = document.getElementsByClassName('indexcheck'); + Array.from(indices).forEach(function(item, index){ + if($(item).prop("checked") == true){ + var id = 'hidden_index_del_' + item.value; $('#form-deleteindex').append(''); + } else { + var id = 'hidden_index_del_' + item.value; + $('#' + id).remove(); } - } else { - var id = 'hidden_index_del_' + item.value; - $('#' + id).remove(); - } + }); } function checkSelected() { diff --git a/diskover-web/public/nav.php b/diskover-web/public/nav.php index 31246ad7..c5b1f236 100644 --- a/diskover-web/public/nav.php +++ b/diskover-web/public/nav.php @@ -322,7 +322,7 @@ search($searchParams); } catch (Missing404Exception $e) { - handleError("Selected indices are no longer available."); + handleError("Selected indices are no longer available. Please select a different index."); } catch (Exception $e) { - handleError('ES error: ' . $e->getMessage(), false); + // reset sort order and reload page if error contains reason that we can not sort + $error_arr = json_decode($e->getMessage(), true); + $error_reason = $error_arr['error']['root_cause'][0]['reason']; + if (strpos($error_reason, "No mapping found for") !== false) { + resetSort('nomapping'); + } elseif (strpos($error_reason, "Text fields are not optimised") !== false) { + resetSort('textfield'); + } else { + handleError('ES error: ' . $e->getMessage(), false); + } } // set total hits @@ -188,6 +196,11 @@ } } arsort($ext_onpage); + // clear current scroll window + if (!empty($scroll_id)) { + $client->clearScroll(array('scroll_id' => $scroll_id)); + $scroll_id = null; + } // end loop break; } @@ -197,7 +210,7 @@ $queryResponse = $client->scroll([ "body" => [ "scroll_id" => $scroll_id, - "scroll" => "1m" + "scroll" => "30s" ] ]); diff --git a/diskover-web/public/selectindices.php b/diskover-web/public/selectindices.php index d6844c12..1972ae34 100644 --- a/diskover-web/public/selectindices.php +++ b/diskover-web/public/selectindices.php @@ -401,7 +401,7 @@ function gtag(){dataLayer.push(arguments);} echo " "; if (!$disabled) { - echo ""; + echo ""; } else { echo ""; } diff --git a/diskover-web/src/diskover/Diskover.php b/diskover-web/src/diskover/Diskover.php index 510ad38a..5d5bb048 100644 --- a/diskover-web/src/diskover/Diskover.php +++ b/diskover-web/src/diskover/Diskover.php @@ -94,7 +94,9 @@ function getIndicesInfoCurl() 'total', 'used', 'free', + 'free_percent', 'available', + 'available_percent', 'file_size', 'file_size_du', 'file_count', @@ -114,6 +116,15 @@ function getIndicesInfoCurl() if (!in_array($fieldname, $field_exclusions)) { $fields[] = $fieldname; } + // check for multi-field add add additional sub-fields fieldname.subfield + if (array_key_exists('properties', $val['mappings']['properties'][$fieldname])) { + foreach ($val['mappings']['properties'][$fieldname]['properties'] as $k => $v) { + $field = $fieldname . '.' . $k; + if (!in_array($field, $field_exclusions)) { + $fields[] = $field; + } + } + } } $indices_curl_info_data[$key] = [ 'uuid' => $val['settings']['index']['uuid'], diff --git a/diskover-web/src/diskover/version.php b/diskover-web/src/diskover/version.php index 6486d04e..6dfee0c5 100644 --- a/diskover-web/src/diskover/version.php +++ b/diskover-web/src/diskover/version.php @@ -17,4 +17,4 @@ */ // diskover-web version -$VERSION = '2.0.6 community edition (ce)'; \ No newline at end of file +$VERSION = '2.0.7 community edition (ce)'; \ No newline at end of file diff --git a/diskover/CHANGELOG.md b/diskover/CHANGELOG.md index d2d99b84..139bd9ad 100644 --- a/diskover/CHANGELOG.md +++ b/diskover/CHANGELOG.md @@ -1,5 +1,14 @@ # Diskover v2 Community Edition Change Log +# [2.0.7] - 2023-01-08 +### fixed +- python error when indexing spaceinfo doc and total disk space > max size for ES long field mapping (AWS storage gateway) +- exception handling for Elasticsearch exception TransportError during bulk uploads +- exception handling for close function call for plugins and alt scanners +### added +- free_percent and available_percent to spaceinfo doc and to es index mappings + + # [2.0.6] - 2022-11-06 ### changed - better handling of errors when importing alternate scanner modules diff --git a/diskover/diskover.py b/diskover/diskover.py index 3cc9a803..62f9c857 100644 --- a/diskover/diskover.py +++ b/diskover/diskover.py @@ -31,6 +31,7 @@ from queue import Queue from random import choice from elasticsearch.helpers.errors import BulkIndexError +from elasticsearch.exceptions import TransportError from diskover_elasticsearch import elasticsearch_connection, \ check_index_exists, create_index, bulk_upload, tune_index @@ -40,7 +41,7 @@ get_file_name, load_plugins, list_plugins, get_plugins_info, set_times, \ get_mem_usage, get_win_path, rem_win_path -version = '2.0.6 community edition (ce)' +version = '2.0.7 community edition (ce)' __version__ = version # Windows check @@ -245,13 +246,22 @@ def close_app(): # close any plugins if plugins_enabled and plugins: for plugin in plugins: - plugin.close(globals()) + try: + plugin.close(globals()) + except AttributeError: + pass + except Exception as e: + logger.exception(e, exc_info=1) + if logtofile: logger_warn.exception(e, exc_info=1) # alt scanner close if alt_scanner: try: alt_scanner.close(globals()) except AttributeError: pass + except Exception as e: + logger.exception(e, exc_info=1) + if logtofile: logger_warn.exception(e, exc_info=1) # if any warnings, exit with custom exit code 64 to indicate index finished but with warnings if warnings > 0: sys.exit(64) @@ -263,13 +273,25 @@ def close_app_critical_error(): # close any plugins if plugins_enabled and plugins: for plugin in plugins: - plugin.close(globals()) + try: + plugin.close(globals()) + except AttributeError: + pass + except Exception as e: + logger.exception(e, exc_info=1) + if logtofile: logger_warn.exception(e, exc_info=1) # alt scanner close if alt_scanner: try: alt_scanner.close(globals()) except AttributeError: pass + except Exception as e: + logger.exception(e, exc_info=1) + if logtofile: logger_warn.exception(e, exc_info=1) + logmsg = 'CRITICAL ERROR EXITING' + logger.critical(logmsg) + if logtofile: logger_warn.critical(logmsg) os._exit(1) @@ -285,8 +307,8 @@ def start_bulk_upload(thread, root, docs): es_upload_start = time.time() try: bulk_upload(es, options.index, docs) - except BulkIndexError as e: - logmsg = '[{0}] FATAL ERROR: Elasticsearch bulk index error! ({1})'.format(thread, e) + except (BulkIndexError, TransportError) as e: + logmsg = '[{0}] FATAL ERROR: Elasticsearch bulk index/transport error! ({1})'.format(thread, e) logger.critical(logmsg, exc_info=1) if logtofile: logger_warn.critical(logmsg, exc_info=1) close_app_critical_error() diff --git a/diskover/diskover_elasticsearch.py b/diskover/diskover_elasticsearch.py index 95084536..a67c7f9a 100644 --- a/diskover/diskover_elasticsearch.py +++ b/diskover/diskover_elasticsearch.py @@ -372,9 +372,15 @@ def create_index(indexname, es): 'free': { 'type': 'long' }, + 'free_percent': { + 'type': 'float' + }, 'available': { 'type': 'long' }, + 'available_percent': { + 'type': 'float' + }, 'file_size': { 'type': 'long' }, diff --git a/diskover/diskover_helpers.py b/diskover/diskover_helpers.py index 14478663..943cc898 100644 --- a/diskover/diskover_helpers.py +++ b/diskover/diskover_helpers.py @@ -293,12 +293,22 @@ def index_info_crawlstart(es, index, path, start, ver, altscanner): mount_path = path if replacepaths: mount_path = replace_path(mount_path) + # Check if too large for long field mapping used by total + maxlongint = 18446744073709551615 + if total > maxlongint: + total = maxlongint + if free > maxlongint: + free = maxlongint + if available > maxlongint: + available = maxlongint data = { 'path': mount_path, 'total': total, 'used': total - free, 'free': free, + 'free_percent': round((total-(total-free))/total*100, 6), 'available': available, + 'available_percent': round((total-(total-available))/total*100, 6), 'type': 'spaceinfo' } es.index(index=index, body=data) @@ -336,12 +346,22 @@ def index_info_crawlstart(es, index, path, start, ver, altscanner): available = available_bytes.value if replacepaths: mount_path = replace_path(mount_path) + # Check if too large for long field mapping used by total + maxlongint = 18446744073709551615 + if total > maxlongint: + total = maxlongint + if free > maxlongint: + free = maxlongint + if available > maxlongint: + available = maxlongint data = { 'path': mount_path, 'total': total, 'used': total - free, 'free': free, + 'free_percent': round((total-(total-free))/total*100, 6), 'available': available, + 'available_percent': round((total-(total-available))/total*100, 6), 'type': 'spaceinfo' } es.index(index=index, body=data)