From 5d71e07fd364baf242aab9ea169afdf024f37149 Mon Sep 17 00:00:00 2001 From: Hironori Yamamoto Date: Sat, 31 Dec 2022 15:59:55 +0900 Subject: [PATCH] Add some facets --- redash-search-sync/Cargo.toml | 2 + redash-search-sync/src/app.rs | 101 ++++++++++++++--------- redash-search-sync/src/main.rs | 2 +- redash-search-sync/src/redash.rs | 67 ++++++++++++++- redash-search-web/components/HitList.tsx | 65 +++++++++++++-- redash-search-web/next.config.js | 5 ++ redash-search-web/package.json | 2 +- redash-search-web/pages/Search.tsx | 30 +++---- redash-search-web/pages/api/graphql.ts | 61 +++++++++++--- redash-search-web/pages/api/models.ts | 8 ++ redash-search-web/pages/index.tsx | 6 +- redash-search-web/yarn.lock | 8 +- 12 files changed, 264 insertions(+), 93 deletions(-) diff --git a/redash-search-sync/Cargo.toml b/redash-search-sync/Cargo.toml index 64b313d..451bf45 100644 --- a/redash-search-sync/Cargo.toml +++ b/redash-search-sync/Cargo.toml @@ -9,8 +9,10 @@ version = "0.1.0" [dependencies] anyhow = {version = "1.0.58", features = ["backtrace"]} async-trait = "0.1.60" +chrono = { version = "0.4.23", features = ["serde"] } config = "0.13.1" hyper = {version = "0.14", features = ["full"]} +once_cell = "1.17.0" opensearch = "2.0.0" reqwest = {version = "0.11", features = ["json"]} serde = {version = "1.0.140", features = ["derive"]} diff --git a/redash-search-sync/src/app.rs b/redash-search-sync/src/app.rs index 20d7a68..7d08706 100644 --- a/redash-search-sync/src/app.rs +++ b/redash-search-sync/src/app.rs @@ -1,24 +1,73 @@ use anyhow::Result; +use chrono::{DateTime, Local}; +use once_cell::sync::Lazy; use opensearch::BulkParts; use opensearch::{http::request::JsonBody, OpenSearch}; use serde::{Deserialize, Serialize}; -use serde_json::json; +use serde_json::{json, Value}; use crate::configs::Configs; use crate::redash::{self, RedashClient}; const REDASH_INDEX_NAME: &str = "redash"; +static INDEX_CONFIG: Lazy = Lazy::new(|| { + json!({ + "settings": { + "analysis": { + "analyzer": { + "sql_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "lowercase" + ], + "char_filter": [ + "sql_char_filter" + ] + } + }, + "char_filter": { + "sql_char_filter": { + "type": "pattern_replace", + "pattern": "[\\.]", + "replacement": " " + } + } + } + }, + "mappings": { + "properties": { + "query": { + "type": "text", + "analyzer": "sql_analyzer" + }, + "created_at": { + "type": "date", + "format": "date_time" + }, + "updated_at": { + "type": "date", + "format": "date_time" + }, + } + } + }) +}); + #[derive(Serialize, Deserialize, Debug, PartialEq)] struct RedashDocument { id: i32, name: String, query: String, - updated_at: String, - created_at: String, + user_name: String, + user_email: String, + created_at: DateTime, + updated_at: DateTime, data_source_id: i32, + data_source_name: String, + data_source_type: String, tags: Vec, - url: String, } pub struct App { @@ -62,42 +111,9 @@ impl App { .create(opensearch::indices::IndicesCreateParts::Index( REDASH_INDEX_NAME, )) - .body(json!({ - "settings": { - "analysis": { - "analyzer": { - "sql_analyzer": { - "type": "custom", - "tokenizer": "standard", - "filter": [ - "lowercase" - ], - "char_filter": [ - "sql_char_filter" - ] - } - }, - "char_filter": { - "sql_char_filter": { - "type": "pattern_replace", - "pattern": "[\\._]", - "replacement": " " - } - } - } - }, - "mappings": { - "properties": { - "query": { - "type": "text", - "analyzer": "sql_analyzer" - } - } - } - })) + .body(INDEX_CONFIG.clone()) .send() .await?; - if !res.status_code().is_success() { tracing::error!( response = res.text().await.unwrap(), @@ -113,6 +129,7 @@ impl App { // TODO: sync only updated queries pub async fn sync(&self) -> Result<()> { + let data_sources = self.redash_client.get_data_sources().await?; let res = self .redash_client .get_queries(redash::GetQueriesRequest { @@ -123,15 +140,19 @@ impl App { .await?; let mut body: Vec> = Vec::new(); for query in res.results { + let data_source = data_sources.get(&query.data_source_id).unwrap(); let doc = RedashDocument { id: query.id, name: query.name, query: query.query, - updated_at: query.updated_at, + user_name: query.user.name, + user_email: query.user.email, created_at: query.created_at, + updated_at: query.updated_at, data_source_id: query.data_source_id, + data_source_name: data_source.name.clone(), + data_source_type: data_source.r#type.clone(), tags: query.tags, - url: format!("{}/queries/{}", self.configs.redash.url, query.id), }; body.push( json!({ diff --git a/redash-search-sync/src/main.rs b/redash-search-sync/src/main.rs index 33c7885..0059ae8 100644 --- a/redash-search-sync/src/main.rs +++ b/redash-search-sync/src/main.rs @@ -8,7 +8,7 @@ use tracing::Level; #[tokio::main] async fn main() { tracing_subscriber::fmt() - .with_max_level(Level::INFO) + .with_max_level(Level::DEBUG) .with_level(true) .with_target(true) .with_thread_ids(true) diff --git a/redash-search-sync/src/redash.rs b/redash-search-sync/src/redash.rs index a0b8bbe..e452625 100644 --- a/redash-search-sync/src/redash.rs +++ b/redash-search-sync/src/redash.rs @@ -1,11 +1,15 @@ +use std::collections::HashMap; + use anyhow::Result; use async_trait::async_trait; +use chrono::{DateTime, Local}; use serde::{Deserialize, Serialize}; #[async_trait] pub trait RedashClient { async fn get_queries(&self, req: GetQueriesRequest) -> Result; + async fn get_data_sources(&self) -> Result; } #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] @@ -20,16 +24,29 @@ pub struct GetQueriesRequest { pub struct RedashQuery { pub id: i32, pub name: String, + pub description: Option, + pub user: RedashUser, pub query: String, pub query_hash: String, pub is_archived: bool, pub is_draft: bool, - pub updated_at: String, - pub created_at: String, + pub created_at: DateTime, + pub updated_at: DateTime, pub data_source_id: i32, pub tags: Vec, } +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +pub struct RedashUser { + pub id: i32, + pub name: String, + pub email: String, + pub is_disabled: bool, + pub is_invitation_pending: bool, + pub updated_at: String, + pub created_at: String, +} + #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] pub struct GetQueriesResponse { pub count: i32, @@ -38,6 +55,20 @@ pub struct GetQueriesResponse { pub results: Vec, } +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +pub struct RedashDataSource { + pub id: i32, + pub name: String, + pub r#type: String, + pub syntax: String, + pub paused: i32, + pub pause_reason: Option, + pub supports_auto_limit: bool, + pub view_only: bool, +} + +type GetDataSourcesResponse = HashMap; + /// This is a default implementation of RedashClient /// NOTE: some fields are omitted for simplicity /// @@ -73,11 +104,41 @@ impl RedashClient for DefaultRedashClient { let res = req.send().await?; if res.status().is_success() { - Ok(res.json().await?) + // TODO: sometimes status is 200 but response shows error + // e.g. "{"took":6,"errors":true, ...}" + // so we need to handle this case + let data = res.json().await?; + tracing::debug!(data = serde_json::to_string(&data).unwrap(), "Got queries"); + Ok(data) } else { let data = res.text().await.unwrap(); tracing::error!(data = data, "Failed to get queries"); Err(anyhow::anyhow!("Failed to get queries")) } } + + async fn get_data_sources(&self) -> Result { + let client = reqwest::Client::new(); + let req = client + .get(format!("{}/api/data_sources", self.base_url)) + .header("Authorization", format!("Key {}", self.api_key)); + + let res = req.send().await?; + if res.status().is_success() { + let data = res.json::>().await?; + tracing::debug!( + data = serde_json::to_string(&data).unwrap(), + "Got data sources" + ); + let data = data + .into_iter() + .map(|ds| (ds.id, ds)) + .collect::(); + Ok(data) + } else { + let data = res.text().await.unwrap(); + tracing::error!(data = data, "Failed to get data sources"); + Err(anyhow::anyhow!("Failed to get data sources")) + } + } } diff --git a/redash-search-web/components/HitList.tsx b/redash-search-web/components/HitList.tsx index 3c8d03b..38aa4c0 100644 --- a/redash-search-web/components/HitList.tsx +++ b/redash-search-web/components/HitList.tsx @@ -3,16 +3,16 @@ import { EuiFlexGrid, EuiFlexItem, EuiCard, - EuiFlexGroup, - EuiTitle, - EuiText, EuiSpacer, - EuiButtonEmpty, - EuiButton, - EuiLink, + EuiDescriptionList, + EuiFlexGroup, + EuiAvatar, } from "@elastic/eui"; import { IResultHitItem } from "../pages/api/models"; import { HighlightedQuery } from "./HighlightedQuery"; +import getConfig from "next/config"; + +const { publicRuntimeConfig } = getConfig(); export interface HitListProps { hitItems: IResultHitItem[]; @@ -25,9 +25,60 @@ const HitsList: React.FC = ({ hitItems }) => { + } title={hit.fields.name} - href={hit.fields.url} + href={`${publicRuntimeConfig.redashURL}/queries/${hit.id}`} + description={hit.fields.description} > + + + + + + + + + + {hit.highlight.query && ( <> diff --git a/redash-search-web/next.config.js b/redash-search-web/next.config.js index 70ba11c..583ec1f 100644 --- a/redash-search-web/next.config.js +++ b/redash-search-web/next.config.js @@ -1,9 +1,14 @@ +const { processEnv } = require("@next/env"); + /** @type {import('next').NextConfig} */ const nextConfig = { reactStrictMode: true, compiler: { emotion: true, }, + publicRuntimeConfig: { + redashURL: processEnv.REDASH__URL.replace(/\/$/, ""), + }, }; module.exports = nextConfig; diff --git a/redash-search-web/package.json b/redash-search-web/package.json index a365866..1ef0e72 100644 --- a/redash-search-web/package.json +++ b/redash-search-web/package.json @@ -30,7 +30,7 @@ "graphql": "16.6.0", "micro": "^10.0.1", "micro-cors": "^0.1.1", - "moment": "2.29.1", + "moment": "^2.29.4", "next": "13.1.1", "next-with-apollo": "5.1.0", "react": "18.2.0", diff --git a/redash-search-web/pages/Search.tsx b/redash-search-web/pages/Search.tsx index a57afc4..f161d7a 100644 --- a/redash-search-web/pages/Search.tsx +++ b/redash-search-web/pages/Search.tsx @@ -1,37 +1,22 @@ -import { useQuery, gql, useLazyQuery } from "@apollo/client"; -import { - useSearchkit, - useSearchkitQueryValue, - useSearchkitVariables, -} from "@searchkit/client"; +import { gql, useLazyQuery } from "@apollo/client"; +import { useSearchkitVariables } from "@searchkit/client"; import { FacetsList, SearchBar, Pagination, SelectedFilters, - SortingSelector, ResetSearchButton, } from "@searchkit/elastic-ui"; -import { useEffect, useState } from "react"; +import { useEffect } from "react"; import { - EuiPage, - EuiPageBody, EuiPageHeaderSection, EuiTitle, EuiHorizontalRule, - EuiButtonGroup, EuiFlexGroup, - EuiFlexItem, EuiPageTemplate, - EuiSearchBar, - EuiHeader, - EuiHeaderSection, - EuiPageSection, EuiSpacer, } from "@elastic/eui"; -import { initializeApollo } from "../lib/apolloClient"; import HitsList from "../components/HitList"; -import { EXPORT_MARKER } from "next/dist/shared/lib/constants"; export const RESULT_SET_QUERY = gql` query resultSet( @@ -82,8 +67,15 @@ export const RESULT_SET_QUERY = gql` id fields { name + description query - url + user_name + user_email + tags + created_at + updated_at + data_source_name + data_source_type } highlight { name diff --git a/redash-search-web/pages/api/graphql.ts b/redash-search-web/pages/api/graphql.ts index 79a31b5..3076d3c 100644 --- a/redash-search-web/pages/api/graphql.ts +++ b/redash-search-web/pages/api/graphql.ts @@ -4,20 +4,26 @@ import Cors from "micro-cors"; import { MultiMatchQuery, RefinementSelectFacet, - RangeFacet, SearchkitSchema, - DateRangeFacet, - SearchkitResolver, - GeoBoundingBoxFilter, - HierarchicalMenuFacet, } from "@searchkit/schema"; -import { json } from "micro"; const searchkitConfig = { host: "http://localhost:9200", + credential: {}, index: "redash", hits: { - fields: ["name", "query", "url", "tags"], + fields: [ + "name", + "query", + "user_name", + "user_email", + "description", + "tags", + "created_at", + "updated_at", + "data_source_name", + "data_source_type", + ], highlightedFields: [ "name", { @@ -39,6 +45,30 @@ const searchkitConfig = { ], query: new MultiMatchQuery({ fields: ["name", "query"] }), facets: [ + new RefinementSelectFacet({ + field: "data_source_type.keyword", + identifier: "data_source_type", + label: "DataSourceType", + multipleSelect: true, + }), + new RefinementSelectFacet({ + field: "data_source_name.keyword", + identifier: "data_source_name", + label: "DataSourceName", + multipleSelect: true, + }), + new RefinementSelectFacet({ + field: "user_name.keyword", + identifier: "user_name", + label: "UserName", + multipleSelect: true, + }), + new RefinementSelectFacet({ + field: "user_email.keyword", + identifier: "user_email", + label: "UserEmail", + multipleSelect: true, + }), new RefinementSelectFacet({ field: "tags.keyword", identifier: "tags", @@ -70,8 +100,17 @@ const server = new ApolloServer({ type HitFields { name: String + description: String query: String - url: String + user_name: String + user_email: String + created_at: String + updated_at: String + crated_date: String + updated_date: String + tags: [String] + data_source_name: String + data_source_type: String } type HitHighlight { @@ -87,11 +126,7 @@ const server = new ApolloServer({ `, ...typeDefs, ], - resolvers: withSearchkitResolvers({ - ResultHit: { - // highlight: (hit: any) => JSON.stringify(hit.highlight), - }, - }), + resolvers: withSearchkitResolvers({}), introspection: true, context: { ...context, diff --git a/redash-search-web/pages/api/models.ts b/redash-search-web/pages/api/models.ts index 2b88a97..acaae55 100644 --- a/redash-search-web/pages/api/models.ts +++ b/redash-search-web/pages/api/models.ts @@ -1,7 +1,15 @@ export interface IResultHitField { name: string; + description: string; query: string; url: string; + user_name: string; + user_email: string; + tags: string[]; + created_at: string; + updated_at: string; + data_source_name: string; + data_source_type: string; } export interface IResultHitHighlight { diff --git a/redash-search-web/pages/index.tsx b/redash-search-web/pages/index.tsx index 8277ba9..0e526a4 100644 --- a/redash-search-web/pages/index.tsx +++ b/redash-search-web/pages/index.tsx @@ -1,9 +1,5 @@ import dynamic from "next/dynamic"; -import { - useSearchkitVariables, - withSearchkit, - withSearchkitRouting, -} from "@searchkit/client"; +import { withSearchkit, withSearchkitRouting } from "@searchkit/client"; const Search = dynamic(() => import("./Search"), { ssr: false, diff --git a/redash-search-web/yarn.lock b/redash-search-web/yarn.lock index 7a4f99d..99a2f61 100644 --- a/redash-search-web/yarn.lock +++ b/redash-search-web/yarn.lock @@ -4700,10 +4700,10 @@ minimist@^1.2.0, minimist@^1.2.6: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== -moment@2.29.1: - version "2.29.1" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" - integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== +moment@^2.29.4: + version "2.29.4" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" + integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== ms@2.0.0: version "2.0.0"