From 8e61b77d3f849661b7175544f471119042fe9551 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 9 Apr 2019 18:38:54 +0300 Subject: [PATCH] SQL: Fix catalog filtering in SYS COLUMNS (#40583) Properly treat '%' as a wildcard for catalog filtering instead of doing a straight string match. Table filtering now considers aliases as well. Add escaping char for LIKE queries with user defined params Fix monotony of ORDINAL_POSITION Add integration test for SYS COLUMNS - currently running only inside single_node since the cluster name is test dependent. Add pattern unescaping for index names Fix #40582 --- .../xpack/sql/jdbc/JdbcDatabaseMetaData.java | 23 +-- .../xpack/sql/qa/security/JdbcSecurityIT.java | 10 +- .../sql/qa/single_node/JdbcCsvSpecIT.java | 16 ++ .../xpack/sql/qa/jdbc/CsvTestUtils.java | 2 + .../xpack/sql/qa/jdbc/JdbcTestUtils.java | 3 +- .../qa/jdbc/SpecBaseIntegrationTestCase.java | 6 +- .../setup_mock_metadata_get_columns.sql | 2 +- .../single-node-only/command-sys.csv-spec | 120 +++++++++++++ .../sql/analysis/index/IndexResolver.java | 17 +- .../plan/logical/command/sys/SysColumns.java | 56 ++++-- .../xpack/sql/util/StringUtils.java | 28 +++ .../logical/command/sys/SysColumnsTests.java | 169 ++++++++++++++---- .../logical/command/sys/SysParserTests.java | 163 ----------------- .../xpack/sql/util/LikeConversionTests.java | 28 ++- 14 files changed, 409 insertions(+), 234 deletions(-) create mode 100644 x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec delete mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java index eaececff16d7c..4d5b6eae2e1e6 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDatabaseMetaData.java @@ -33,6 +33,8 @@ */ class JdbcDatabaseMetaData implements DatabaseMetaData, JdbcWrapper { + private static final String WILDCARD = "%"; + private final JdbcConnection con; JdbcDatabaseMetaData(JdbcConnection con) { @@ -713,19 +715,19 @@ private boolean isDefaultCatalog(String catalog) throws SQLException { // null means catalog info is irrelevant // % means return all catalogs // EMPTY means return those without a catalog - return catalog == null || catalog.equals(EMPTY) || catalog.equals("%") || catalog.equals(defaultCatalog()); + return catalog == null || catalog.equals(EMPTY) || catalog.equals(WILDCARD) || catalog.equals(defaultCatalog()); } private boolean isDefaultSchema(String schema) { // null means schema info is irrelevant // % means return all schemas` // EMPTY means return those without a schema - return schema == null || schema.equals(EMPTY) || schema.equals("%"); + return schema == null || schema.equals(EMPTY) || schema.equals(WILDCARD); } @Override public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { - String statement = "SYS TABLES CATALOG LIKE ? LIKE ?"; + String statement = "SYS TABLES CATALOG LIKE ? ESCAPE '\\' LIKE ? ESCAPE '\\' "; if (types != null && types.length > 0) { statement += " TYPE ?"; @@ -738,8 +740,8 @@ public ResultSet getTables(String catalog, String schemaPattern, String tableNam } PreparedStatement ps = con.prepareStatement(statement); - ps.setString(1, catalog != null ? catalog.trim() : "%"); - ps.setString(2, tableNamePattern != null ? tableNamePattern.trim() : "%"); + ps.setString(1, catalog != null ? catalog.trim() : WILDCARD); + ps.setString(2, tableNamePattern != null ? tableNamePattern.trim() : WILDCARD); if (types != null && types.length > 0) { for (int i = 0; i < types.length; i++) { @@ -784,14 +786,15 @@ public ResultSet getTableTypes() throws SQLException { return memorySet(con.cfg, columnInfo("TABLE_TYPES", "TABLE_TYPE"), data); } + @Override public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { - PreparedStatement ps = con.prepareStatement("SYS COLUMNS CATALOG ? TABLE LIKE ? LIKE ?"); - // TODO: until passing null works, pass an empty string - ps.setString(1, catalog != null ? catalog.trim() : EMPTY); - ps.setString(2, tableNamePattern != null ? tableNamePattern.trim() : "%"); - ps.setString(3, columnNamePattern != null ? columnNamePattern.trim() : "%"); + PreparedStatement ps = con.prepareStatement("SYS COLUMNS CATALOG ? TABLE LIKE ? ESCAPE '\\' LIKE ? ESCAPE '\\'"); + // NB: catalog is not a pattern hence why null is send instead + ps.setString(1, catalog != null ? catalog.trim() : null); + ps.setString(2, tableNamePattern != null ? tableNamePattern.trim() : WILDCARD); + ps.setString(3, columnNamePattern != null ? columnNamePattern.trim() : WILDCARD); return ps.executeQuery(); } diff --git a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSecurityIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSecurityIT.java index c56f3b23946e7..a911e7d4854ae 100644 --- a/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSecurityIT.java +++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSecurityIT.java @@ -232,13 +232,13 @@ public void expectUnknownColumn(String user, String sql, String column) throws E public void checkNoMonitorMain(String user) throws Exception { // Without monitor/main the JDBC driver - ES server version comparison doesn't take place, which fails everything else expectUnauthorized("cluster:monitor/main", user, () -> es(userProperties(user))); - expectUnauthorized("cluster:monitor/main", user, () -> es(userProperties(user)).getMetaData().getDatabaseMajorVersion()); + expectUnauthorized("cluster:monitor/main", user, () -> es(userProperties(user)).getMetaData().getDatabaseMajorVersion()); expectUnauthorized("cluster:monitor/main", user, () -> es(userProperties(user)).getMetaData().getDatabaseMinorVersion()); - expectUnauthorized("cluster:monitor/main", user, + expectUnauthorized("cluster:monitor/main", user, () -> es(userProperties(user)).createStatement().executeQuery("SELECT * FROM test")); - expectUnauthorized("cluster:monitor/main", user, + expectUnauthorized("cluster:monitor/main", user, () -> es(userProperties(user)).createStatement().executeQuery("SHOW TABLES LIKE 'test'")); - expectUnauthorized("cluster:monitor/main", user, + expectUnauthorized("cluster:monitor/main", user, () -> es(userProperties(user)).createStatement().executeQuery("DESCRIBE test")); } @@ -292,7 +292,7 @@ public void testMetaDataGetColumnsWorksAsFullAccess() throws Exception { expectActionMatchesAdmin( con -> con.getMetaData().getColumns(null, "%", "%t", "%"), "full_access", - con -> con.getMetaData().getColumns(null, "%", "%", "%")); + con -> con.getMetaData().getColumns(null, "%", "%t", "%")); } public void testMetaDataGetColumnsWithNoAccess() throws Exception { diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java index 66ac2e2c7df24..f742b1304a79e 100644 --- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java @@ -5,10 +5,26 @@ */ package org.elasticsearch.xpack.sql.qa.single_node; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.xpack.sql.qa.jdbc.CsvSpecTestCase; import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser; + public class JdbcCsvSpecIT extends CsvSpecTestCase { + + + @ParametersFactory(argumentFormatting = PARAM_FORMATTING) + public static List readScriptSpec() throws Exception { + List list = new ArrayList<>(); + list.addAll(CsvSpecTestCase.readScriptSpec()); + return readScriptSpec("/single-node-only/command-sys.csv-spec", specParser()); + } + public JdbcCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) { super(fileName, groupName, testName, lineNumber, testCase); } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java index 8cc8cf6e04044..6376bd13308d6 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java @@ -155,6 +155,8 @@ private static String resolveColumnType(String type) { return "timestamp"; case "bt": return "byte"; + case "sh": + return "short"; default: return type; } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index 19c30b55e92b1..123f22073ae57 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -212,7 +212,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO @SuppressForbidden(reason = "need to open jar") private static JarInputStream getJarStream(URL resource) throws IOException { URLConnection con = resource.openConnection(); - con.setDefaultUseCaches(false); + // do not to cache files (to avoid keeping file handles around) + con.setUseCaches(false); return new JarInputStream(con.getInputStream()); } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java index 4282b97d87fb1..05ba49bbd0d32 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; +import java.net.URLConnection; import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; @@ -216,6 +217,9 @@ public interface Parser { @SuppressForbidden(reason = "test reads from jar") public static InputStream readFromJarUrl(URL source) throws IOException { - return source.openStream(); + URLConnection con = source.openConnection(); + // do not to cache files (to avoid keeping file handles around) + con.setUseCaches(false); + return con.getInputStream(); } } diff --git a/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_columns.sql b/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_columns.sql index d6df2fbb9e14b..5e02df28b068d 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_columns.sql +++ b/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_columns.sql @@ -30,7 +30,7 @@ FROM DUAL UNION ALL SELECT null, 'test1', 'name.keyword', 12, 'KEYWORD', 32766, 2147483647, null, null, 1, -- columnNullable - null, null, 12, 0, 2147483647, 1, 'YES', null, null, null, null, 'NO', 'NO' + null, null, 12, 0, 2147483647, 2, 'YES', null, null, null, null, 'NO', 'NO' FROM DUAL UNION ALL SELECT null, 'test2', 'date', 93, 'DATETIME', 29, 8, null, null, diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec new file mode 100644 index 0000000000000..f6b02ba4bea43 --- /dev/null +++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec @@ -0,0 +1,120 @@ +// +// Sys Commands +// + +sysColumnsWithTableLikeWithEscape +SYS COLUMNS TABLE LIKE 'test\_emp' ESCAPE '\'; + + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO +; + +sysColumnsWithTableLikeNoEscape +SYS COLUMNS TABLE LIKE 'test_emp'; + +// since there's no escaping test_emp means test*emp which matches also test_alias_emp +// however as there's no way to filter the matching indices, we can't exclude the field + + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +; + +sysColumnsWithCatalogAndLike +SYS COLUMNS CATALOG 'x-pack_plugin_sql_qa_single-node_integTestCluster' TABLE LIKE 'test\_emp\_copy' ESCAPE '\'; + + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +---------------+---------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy|salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +; + +sysColumnsOnAliasWithTableLike +SYS COLUMNS TABLE LIKE 'test\_alias' ESCAPE '\'; + + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +; + +sysColumnsAllTables +SYS COLUMNS TABLE LIKE '%'; + + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +x-pack_plugin_sql_qa_single-node_integTestCluster |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |logs |client_ip |12 |IP |0 |39 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |logs |dest_ip |12 |IP |0 |39 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +x-pack_plugin_sql_qa_single-node_integTestCluster |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +; \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java index ec2dfa46f47f2..367c9ea3a149f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java @@ -415,6 +415,8 @@ public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, Ac GetIndexRequest getIndexRequest = createGetIndexRequest(indexWildcard); client.admin().indices().getIndex(getIndexRequest, ActionListener.wrap(getIndexResponse -> { ImmutableOpenMap> mappings = getIndexResponse.getMappings(); + ImmutableOpenMap> aliases = getIndexResponse.getAliases(); + List results = new ArrayList<>(mappings.size()); Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null; for (ObjectObjectCursor> indexMappings : mappings) { @@ -425,7 +427,20 @@ public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, Ac * and not the concrete index: there is a well known information leak of the concrete index name in the response. */ String concreteIndex = indexMappings.key; - if (pattern == null || pattern.matcher(concreteIndex).matches()) { + + // take into account aliases + List aliasMetadata = aliases.get(concreteIndex); + boolean matchesAlias = false; + if (pattern != null && aliasMetadata != null) { + for (AliasMetaData aliasMeta : aliasMetadata) { + if (pattern.matcher(aliasMeta.alias()).matches()) { + matchesAlias = true; + break; + } + } + } + + if (pattern == null || matchesAlias || pattern.matcher(concreteIndex).matches()) { IndexResolution getIndexResult = buildGetIndexResult(concreteIndex, concreteIndex, indexMappings.value); if (getIndexResult.isValid()) { results.add(getIndexResult.get()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java index e5c80197296c2..68cfefe7fb572 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumns.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.sql.plan.logical.command.sys; +import org.apache.lucene.util.Counter; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.sql.analysis.index.EsIndex; @@ -20,6 +21,7 @@ import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypes; import org.elasticsearch.xpack.sql.type.EsField; +import org.elasticsearch.xpack.sql.util.StringUtils; import java.sql.DatabaseMetaData; import java.util.ArrayList; @@ -101,40 +103,63 @@ public void execute(SqlSession session, ActionListener listener) { String cluster = session.indexResolver().clusterName(); // bail-out early if the catalog is present but differs - if (Strings.hasText(catalog) && !cluster.equals(catalog)) { + if (Strings.hasText(catalog) && cluster.equals(catalog) == false) { listener.onResponse(Rows.empty(output)); return; } + // save original index name (as the pattern can contain special chars) + String indexName = index != null ? index : (pattern != null ? StringUtils.likeToUnescaped(pattern.pattern(), + pattern.escape()) : ""); String idx = index != null ? index : (pattern != null ? pattern.asIndexNameWildcard() : "*"); String regex = pattern != null ? pattern.asJavaRegex() : null; Pattern columnMatcher = columnPattern != null ? Pattern.compile(columnPattern.asJavaRegex()) : null; - session.indexResolver().resolveAsSeparateMappings(idx, regex, ActionListener.wrap(esIndices -> { - List> rows = new ArrayList<>(); - for (EsIndex esIndex : esIndices) { - fillInRows(cluster, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode); - } + // special case fo '%' (translated to *) + if ("*".equals(idx)) { + session.indexResolver().resolveAsSeparateMappings(idx, regex, ActionListener.wrap(esIndices -> { + List> rows = new ArrayList<>(); + for (EsIndex esIndex : esIndices) { + fillInRows(cluster, esIndex.name(), esIndex.mapping(), null, rows, columnMatcher, mode); + } + + listener.onResponse(Rows.of(output, rows)); + }, listener::onFailure)); + } + // otherwise use a merged mapping + else { + session.indexResolver().resolveAsMergedMapping(idx, regex, ActionListener.wrap(r -> { + List> rows = new ArrayList<>(); + // populate the data only when a target is found + if (r.isValid() == true) { + EsIndex esIndex = r.get(); + fillInRows(cluster, indexName, esIndex.mapping(), null, rows, columnMatcher, mode); + } - listener.onResponse(Rows.of(output, rows)); - }, listener::onFailure)); + listener.onResponse(Rows.of(output, rows)); + }, listener::onFailure)); + } } static void fillInRows(String clusterName, String indexName, Map mapping, String prefix, List> rows, Pattern columnMatcher, Mode mode) { - int pos = 0; + fillInRows(clusterName, indexName, mapping, prefix, rows, columnMatcher, Counter.newCounter(), mode); + } + + private static void fillInRows(String clusterName, String indexName, Map mapping, String prefix, List> rows, + Pattern columnMatcher, Counter position, Mode mode) { boolean isOdbcClient = mode == Mode.ODBC; for (Map.Entry entry : mapping.entrySet()) { - pos++; // JDBC is 1-based so we start with 1 here + position.addAndGet(1); // JDBC is 1-based so we start with 1 here String name = entry.getKey(); name = prefix != null ? prefix + "." + name : name; EsField field = entry.getValue(); DataType type = field.getDataType(); - // skip the nested, object and unsupported types for JDBC and ODBC - if (type.isPrimitive() || false == Mode.isDriver(mode)) { + // skip the nested, object and unsupported types + if (type.isPrimitive()) { if (columnMatcher == null || columnMatcher.matcher(name).matches()) { rows.add(asList(clusterName, // schema is not supported @@ -162,7 +187,7 @@ static void fillInRows(String clusterName, String indexName, Map mapping = TypesTests.loadMapping("mapping-multi-field-with-nested.json", true); + private final IndexInfo index = new IndexInfo("test_emp", IndexType.INDEX); + private final IndexInfo alias = new IndexInfo("alias", IndexType.ALIAS); + + public void testSysColumns() { List> rows = new ArrayList<>(); SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, randomValueOtherThanMany(Mode::isDriver, () -> randomFrom(Mode.values()))); - assertEquals(17, rows.size()); + // nested fields are ignored + assertEquals(13, rows.size()); assertEquals(24, rows.get(0).size()); List row = rows.get(0); @@ -54,81 +91,57 @@ public void testSysColumns() { assertEquals(8, bufferLength(row)); row = rows.get(5); - assertEquals("unsupported", name(row)); - assertEquals(Types.OTHER, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(0, bufferLength(row)); - - row = rows.get(6); - assertEquals("some", name(row)); - assertEquals(Types.STRUCT, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(-1, bufferLength(row)); - - row = rows.get(7); - assertEquals("some.dotted", name(row)); - assertEquals(Types.STRUCT, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(-1, bufferLength(row)); - - row = rows.get(8); assertEquals("some.dotted.field", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - - row = rows.get(9); + + row = rows.get(6); assertEquals("some.string", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(10); + row = rows.get(7); assertEquals("some.string.normalized", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(11); + row = rows.get(8); assertEquals("some.string.typical", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(12); + row = rows.get(9); assertEquals("some.ambiguous", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(13); + row = rows.get(10); assertEquals("some.ambiguous.one", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(14); + row = rows.get(11); assertEquals("some.ambiguous.two", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - row = rows.get(15); + row = rows.get(12); assertEquals("some.ambiguous.normalized", name(row)); assertEquals(Types.VARCHAR, sqlType(row)); assertEquals(null, radix(row)); assertEquals(Integer.MAX_VALUE, bufferLength(row)); - - row = rows.get(16); - assertEquals("foo_type", name(row)); - assertEquals(Types.OTHER, sqlType(row)); - assertEquals(null, radix(row)); - assertEquals(0, bufferLength(row)); } public void testSysColumnsInOdbcMode() { List> rows = new ArrayList<>(); - SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, + SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, Mode.ODBC); assertEquals(13, rows.size()); assertEquals(24, rows.get(0).size()); @@ -263,7 +276,7 @@ public void testSysColumnsInOdbcMode() { public void testSysColumnsInJdbcMode() { List> rows = new ArrayList<>(); - SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, + SysColumns.fillInRows("test", "index", TypesTests.loadMapping("mapping-multi-field-variation.json", true), null, rows, null, Mode.JDBC); assertEquals(13, rows.size()); assertEquals(24, rows.get(0).size()); @@ -431,4 +444,88 @@ private static Object sqlDataType(List list) { private static Object sqlDataTypeSub(List list) { return list.get(14); } -} + + public void testSysColumnsNoArg() throws Exception { + executeCommand("SYS COLUMNS", emptyList(), r -> { + assertEquals(13, r.size()); + assertEquals(CLUSTER_NAME, r.column(0)); + // no index specified + assertEquals("", r.column(2)); + assertEquals("bool", r.column(3)); + r.advanceRow(); + assertEquals(CLUSTER_NAME, r.column(0)); + // no index specified + assertEquals("", r.column(2)); + assertEquals("int", r.column(3)); + }, mapping); + } + + public void testSysColumnsWithCatalogWildcard() throws Exception { + executeCommand("SYS COLUMNS CATALOG 'cluster' TABLE LIKE 'test' LIKE '%'", emptyList(), r -> { + assertEquals(13, r.size()); + assertEquals(CLUSTER_NAME, r.column(0)); + assertEquals("test", r.column(2)); + assertEquals("bool", r.column(3)); + r.advanceRow(); + assertEquals(CLUSTER_NAME, r.column(0)); + assertEquals("test", r.column(2)); + assertEquals("int", r.column(3)); + }, mapping); + } + + public void testSysColumnsWithMissingCatalog() throws Exception { + executeCommand("SYS COLUMNS TABLE LIKE 'test' LIKE '%'", emptyList(), r -> { + assertEquals(13, r.size()); + assertEquals(CLUSTER_NAME, r.column(0)); + assertEquals("test", r.column(2)); + assertEquals("bool", r.column(3)); + r.advanceRow(); + assertEquals(CLUSTER_NAME, r.column(0)); + assertEquals("test", r.column(2)); + assertEquals("int", r.column(3)); + }, mapping); + } + + public void testSysColumnsWithNullCatalog() throws Exception { + executeCommand("SYS COLUMNS CATALOG ? TABLE LIKE 'test' LIKE '%'", singletonList(new SqlTypedParamValue("keyword", null)), r -> { + assertEquals(13, r.size()); + assertEquals(CLUSTER_NAME, r.column(0)); + assertEquals("test", r.column(2)); + assertEquals("bool", r.column(3)); + r.advanceRow(); + assertEquals(CLUSTER_NAME, r.column(0)); + assertEquals("test", r.column(2)); + assertEquals("int", r.column(3)); + }, mapping); + } + + @SuppressWarnings({ "unchecked" }) + private void executeCommand(String sql, List params, Consumer consumer, Map mapping) + throws Exception { + Tuple tuple = sql(sql, params, mapping); + + IndexResolver resolver = tuple.v2().indexResolver(); + + EsIndex test = new EsIndex("test", mapping); + + doAnswer(invocation -> { + ((ActionListener) invocation.getArguments()[2]).onResponse(IndexResolution.valid(test)); + return Void.TYPE; + }).when(resolver).resolveAsMergedMapping(any(), any(), any()); + + tuple.v1().execute(tuple.v2(), wrap(consumer::accept, ex -> fail(ex.getMessage()))); + } + + private Tuple sql(String sql, List params, Map mapping) { + EsIndex test = new EsIndex("test", mapping); + Analyzer analyzer = new Analyzer(TestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), + new Verifier(new Metrics())); + Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params), true); + + IndexResolver resolver = mock(IndexResolver.class); + when(resolver.clusterName()).thenReturn(CLUSTER_NAME); + + SqlSession session = new SqlSession(TestUtils.TEST_CFG, null, null, resolver, null, null, null, null, null); + return new Tuple<>(cmd, session); + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java deleted file mode 100644 index 110c320d679e4..0000000000000 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.plan.logical.command.sys; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.sql.TestUtils; -import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer; -import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier; -import org.elasticsearch.xpack.sql.analysis.index.EsIndex; -import org.elasticsearch.xpack.sql.analysis.index.IndexResolution; -import org.elasticsearch.xpack.sql.analysis.index.IndexResolver; -import org.elasticsearch.xpack.sql.expression.function.FunctionRegistry; -import org.elasticsearch.xpack.sql.parser.SqlParser; -import org.elasticsearch.xpack.sql.plan.logical.command.Command; -import org.elasticsearch.xpack.sql.session.SqlSession; -import org.elasticsearch.xpack.sql.stats.Metrics; -import org.elasticsearch.xpack.sql.type.DataType; -import org.elasticsearch.xpack.sql.type.EsField; -import org.elasticsearch.xpack.sql.type.TypesTests; - -import java.util.List; -import java.util.Map; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class SysParserTests extends ESTestCase { - - private final SqlParser parser = new SqlParser(); - private final Map mapping = TypesTests.loadMapping("mapping-multi-field-with-nested.json", true); - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private Tuple sql(String sql) { - EsIndex test = new EsIndex("test", mapping); - Analyzer analyzer = new Analyzer(TestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics())); - Command cmd = (Command) analyzer.analyze(parser.createStatement(sql), true); - - IndexResolver resolver = mock(IndexResolver.class); - when(resolver.clusterName()).thenReturn("cluster"); - - doAnswer(invocation -> { - ((ActionListener) invocation.getArguments()[2]).onResponse(singletonList(test)); - return Void.TYPE; - }).when(resolver).resolveAsSeparateMappings(any(), any(), any()); - - SqlSession session = new SqlSession(TestUtils.TEST_CFG, null, null, resolver, null, null, null, null, null); - return new Tuple<>(cmd, session); - } - - public void testSysTypes() { - Command cmd = sql("SYS TYPES").v1(); - - List names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "FLOAT", "DOUBLE", "SCALED_FLOAT", - "KEYWORD", "TEXT", "IP", "BOOLEAN", "DATE", "TIME", "DATETIME", - "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND", - "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND", - "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND", - "UNSUPPORTED", "OBJECT", "NESTED"); - - cmd.execute(null, ActionListener.wrap(r -> { - assertEquals(19, r.columnCount()); - assertEquals(DataType.values().length, r.size()); - assertFalse(r.schema().types().contains(DataType.NULL)); - // test numeric as signed - assertFalse(r.column(9, Boolean.class)); - // make sure precision is returned as boolean (not int) - assertFalse(r.column(10, Boolean.class)); - // no auto-increment - assertFalse(r.column(11, Boolean.class)); - - for (int i = 0; i < r.size(); i++) { - assertEquals(names.get(i), r.column(0)); - r.advanceRow(); - } - - }, ex -> fail(ex.getMessage()))); - } - - public void testSysColsNoArgs() { - runSysColumns("SYS COLUMNS"); - } - - public void testSysColumnEmptyCatalog() { - Tuple sql = sql("SYS COLUMNS CATALOG '' TABLE LIKE '%' LIKE '%'"); - - sql.v1().execute(sql.v2(), ActionListener.wrap(r -> { - assertEquals(24, r.columnCount()); - assertEquals(22, r.size()); - }, ex -> fail(ex.getMessage()))); - } - - public void testSysColsTableOnlyCatalog() { - Tuple sql = sql("SYS COLUMNS CATALOG 'catalog'"); - - sql.v1().execute(sql.v2(), ActionListener.wrap(r -> { - assertEquals(24, r.columnCount()); - assertEquals(0, r.size()); - }, ex -> fail(ex.getMessage()))); - } - - public void testSysColsTableOnlyPattern() { - runSysColumns("SYS COLUMNS TABLE LIKE 'test'"); - } - - public void testSysColsColOnlyPattern() { - runSysColumns("SYS COLUMNS LIKE '%'"); - } - - public void testSysColsTableAndColsPattern() { - runSysColumns("SYS COLUMNS TABLE LIKE 'test' LIKE '%'"); - } - - - private void runSysColumns(String commandVariation) { - Tuple sql = sql(commandVariation); - List names = asList("bool", - "int", - "text", - "keyword", - "unsupported", - "date", - "some", - "some.dotted", - "some.dotted.field", - "some.string", - "some.string.normalized", - "some.string.typical", - "some.ambiguous", - "some.ambiguous.one", - "some.ambiguous.two", - "some.ambiguous.normalized", - "dep", - "dep.dep_name", - "dep.dep_id", - "dep.dep_id.keyword", - "dep.end_date", - "dep.start_date"); - - sql.v1().execute(sql.v2(), ActionListener.wrap(r -> { - assertEquals(24, r.columnCount()); - assertEquals(22, r.size()); - - for (int i = 0; i < r.size(); i++) { - assertEquals("cluster", r.column(0)); - assertNull(r.column(1)); - assertEquals("test", r.column(2)); - assertEquals(names.get(i), r.column(3)); - r.advanceRow(); - } - - }, ex -> fail(ex.getMessage()))); - } -} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/util/LikeConversionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/util/LikeConversionTests.java index 19a544c14e50b..29cbb9b985ffd 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/util/LikeConversionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/util/LikeConversionTests.java @@ -9,6 +9,7 @@ import static org.elasticsearch.xpack.sql.util.StringUtils.likeToJavaPattern; import static org.elasticsearch.xpack.sql.util.StringUtils.likeToLuceneWildcard; +import static org.elasticsearch.xpack.sql.util.StringUtils.likeToUnescaped; public class LikeConversionTests extends ESTestCase { @@ -20,6 +21,10 @@ private static String wildcard(String pattern) { return likeToLuceneWildcard(pattern, '|'); } + private static String unescape(String pattern) { + return likeToUnescaped(pattern, '|'); + } + public void testNoRegex() { assertEquals("^fooBar$", regex("fooBar")); } @@ -103,4 +108,25 @@ public void testWildcardTripleEscaping() { public void testWildcardIgnoreDoubleEscapedButSkipEscapingOfSql() { assertEquals("foo\\\\\\*bar\\\\?\\?", wildcard("foo\\*bar\\_?")); } -} + + public void testUnescapeLiteral() { + assertEquals("foo", unescape("foo")); + } + + public void testUnescapeEscaped() { + assertEquals("foo_bar", unescape("foo|_bar")); + } + + public void testUnescapeEscapedEscape() { + assertEquals("foo|_bar", unescape("foo||_bar")); + } + + public void testUnescapeLastCharEscape() { + assertEquals("foo_bar|", unescape("foo|_bar|")); + } + + public void testUnescapeMultipleEscapes() { + assertEquals("foo|_bar|", unescape("foo|||_bar||")); + } + +} \ No newline at end of file