From edf5df8194287405dd09588566f4dcd4e81580a5 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Mon, 6 Jan 2025 22:37:12 +0800 Subject: [PATCH] feat: add new SVG icons and enhance spider form functionality - Introduced new SVG icons for Colly, Playwright, Puppeteer, Scrapy, and Selenium to enrich the visual representation of spider templates. - Refactored ScheduleForm.vue and SpiderForm.vue to utilize active nodes for improved node selection in the UI. - Updated RunSpiderDialog.vue to enhance node management and selection logic. - Enhanced internationalization support by adding new labels for template documentation in both English and Chinese. - Improved type definitions to accommodate new features related to spider templates and node management. These changes enhance the user experience by providing a more intuitive interface for managing spider configurations and improving visual consistency across the application. --- src/assets/svg/icons/colly.svg | 12 ++ src/assets/svg/icons/playwright.svg | 1 + src/assets/svg/icons/puppeteer.svg | 18 ++ src/assets/svg/icons/scrapy.svg | 13 ++ src/assets/svg/icons/selenium.svg | 8 + src/components/core/schedule/ScheduleForm.vue | 22 +- .../core/spider/RunSpiderDialog.vue | 5 +- src/components/core/spider/SpiderForm.vue | 193 ++++++++++++------ src/components/core/task/CreateTaskDialog.vue | 6 +- src/i18n/lang/en/components/spider.ts | 1 + src/i18n/lang/zh/components/spider.ts | 1 + src/interfaces/i18n/components/spider.d.ts | 1 + src/interfaces/models/spider.d.ts | 10 + src/utils/spider.ts | 163 ++++++++++----- src/views/dependency/list/DependencyList.vue | 6 + 15 files changed, 338 insertions(+), 122 deletions(-) create mode 100644 src/assets/svg/icons/colly.svg create mode 100644 src/assets/svg/icons/playwright.svg create mode 100644 src/assets/svg/icons/puppeteer.svg create mode 100644 src/assets/svg/icons/scrapy.svg create mode 100644 src/assets/svg/icons/selenium.svg diff --git a/src/assets/svg/icons/colly.svg b/src/assets/svg/icons/colly.svg new file mode 100644 index 0000000000000..f89aea5f2bfa0 --- /dev/null +++ b/src/assets/svg/icons/colly.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/src/assets/svg/icons/playwright.svg b/src/assets/svg/icons/playwright.svg new file mode 100644 index 0000000000000..3dcb4ac029216 --- /dev/null +++ b/src/assets/svg/icons/playwright.svg @@ -0,0 +1 @@ +Playwright Streamline Icon: https://streamlinehq.com \ No newline at end of file diff --git a/src/assets/svg/icons/puppeteer.svg b/src/assets/svg/icons/puppeteer.svg new file mode 100644 index 0000000000000..dd734b182778b --- /dev/null +++ b/src/assets/svg/icons/puppeteer.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/assets/svg/icons/scrapy.svg b/src/assets/svg/icons/scrapy.svg new file mode 100644 index 0000000000000..224fee19cf222 --- /dev/null +++ b/src/assets/svg/icons/scrapy.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/src/assets/svg/icons/selenium.svg b/src/assets/svg/icons/selenium.svg new file mode 100644 index 0000000000000..e9d6043d67488 --- /dev/null +++ b/src/assets/svg/icons/selenium.svg @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/src/components/core/schedule/ScheduleForm.vue b/src/components/core/schedule/ScheduleForm.vue index 75236d41ad7b2..ca23946f03b9d 100644 --- a/src/components/core/schedule/ScheduleForm.vue +++ b/src/components/core/schedule/ScheduleForm.vue @@ -24,7 +24,7 @@ const { } = useSchedule(store); // use node -const { allListSelectOptions: allNodeSelectOptions } = useNode(store); +const { activeNodesSorted: activeNodes } = useNode(store); // use spider const { allListSelectOptions: allSpiderSelectOptions } = useSpider(store); @@ -190,11 +190,23 @@ defineOptions({ name: 'ClScheduleForm' }); :label="t('components.schedule.form.selectedNodes')" required > - + multiple + :placeholder="t('components.schedule.form.selectedNodes')" + > + + + + + {{ n.name }} + + diff --git a/src/components/core/spider/RunSpiderDialog.vue b/src/components/core/spider/RunSpiderDialog.vue index 84f11f33c001d..746ad244a934e 100644 --- a/src/components/core/spider/RunSpiderDialog.vue +++ b/src/components/core/spider/RunSpiderDialog.vue @@ -25,7 +25,7 @@ const store = useStore(); const { activeNodesSorted: activeNodes } = useNode(store); const toRunNodes = computed(() => { - const { mode, node_ids } = form.value; + const { mode, node_ids } = options.value; return getToRunNodes(mode, node_ids, activeNodes.value); }); @@ -47,6 +47,7 @@ const getOptions = (): SpiderRunOptions => { cmd: form.value.cmd, param: form.value.param, priority: form.value.priority || 5, + node_ids: form.value.node_ids || [], }; }; @@ -69,7 +70,6 @@ const title = computed(() => { const onClose = () => { const { ns } = props; store.commit(`${ns}/hideDialog`); - store.commit(`${ns}/resetForm`); }; const onConfirm = async () => { @@ -98,6 +98,7 @@ defineOptions({ name: 'ClRunSpiderDialog' }); :title="title" :visible="visible" class-name="run-spider-dialog" + width="1024px" @close="onClose" @confirm="onConfirm" > diff --git a/src/components/core/spider/SpiderForm.vue b/src/components/core/spider/SpiderForm.vue index a907392b1343c..608086c2c6f3d 100644 --- a/src/components/core/spider/SpiderForm.vue +++ b/src/components/core/spider/SpiderForm.vue @@ -2,12 +2,12 @@ import { computed, ref, watch } from 'vue'; import { useStore } from 'vuex'; import { useSpider, useProject, useNode } from '@/components'; -import { TASK_MODE_SELECTED_NODES } from '@/constants/task'; +import { TASK_MODE_RANDOM, TASK_MODE_SELECTED_NODES } from '@/constants/task'; import pinyin, { STYLE_NORMAL } from 'pinyin'; import { isZeroObjectId } from '@/utils/mongo'; import { useSpiderDetail } from '@/views'; -import { priorityOptions, translate } from '@/utils'; -import { getSpiderTemplates } from '@/utils/spider'; +import { getToRunNodes, priorityOptions, translate } from '@/utils'; +import { getSpiderTemplateGroups, getSpiderTemplates } from '@/utils/spider'; // i18n const t = translate; @@ -16,7 +16,12 @@ const t = translate; const store = useStore(); // use node -const { allListSelectOptions: allNodeSelectOptions } = useNode(store); +const { activeNodesSorted: activeNodes } = useNode(store); + +const toRunNodes = computed(() => { + const { mode, node_ids } = form.value; + return getToRunNodes(mode, node_ids, activeNodes.value); +}); // use project const { allListSelectOptionsWithEmpty: allProjectSelectOptions } = @@ -28,6 +33,8 @@ const { form, formRef, isFormItemDisabled, modeOptions } = useSpider(store); // use spider detail const { activeId } = useSpiderDetail(); +const isDetail = computed(() => !!activeId.value); + // whether col field of form has been changed const isFormColChanged = ref(false); @@ -66,20 +73,26 @@ const validate = async () => { await formRef.value?.validate(); }; -const spiderTemplateOptions = computed(() => { - return getSpiderTemplates().map(({ name, label }) => ({ - label, - value: name, +const spiderTemplateGroupOptions = computed(() => { + return getSpiderTemplateGroups().map(group => ({ + label: group.label, + icon: group.icon || ['fa', 'box'], + children: group.templates.map(({ name, label, icon }) => ({ + label, + value: name, + icon: icon || ['fa', 'box'], + })), })); }); const onTemplateChange = (value: string) => { const template = getSpiderTemplates().find(d => d.name === value); if (!template) return; - if (!form.value.name) { - form.value.name = `${template.name}_spider`; - } + form.value.name = `${template.name}_spider`; form.value.cmd = template.cmd; }; +const activeTemplateOption = computed(() => { + return getSpiderTemplates().find(d => d.name === form.value.template); +}); defineExpose({ validate, @@ -91,21 +104,53 @@ defineOptions({ name: 'ClSpiderForm' }); - - - - - + - + + + + + + + {{ n.name }} + + + + + + + + - - - - - - + + diff --git a/src/components/core/task/CreateTaskDialog.vue b/src/components/core/task/CreateTaskDialog.vue index c0c100fb47d3f..1d8c114f1da50 100644 --- a/src/components/core/task/CreateTaskDialog.vue +++ b/src/components/core/task/CreateTaskDialog.vue @@ -15,11 +15,6 @@ const { form, createEditDialogVisible } = useTask(store); const visible = computed(() => createEditDialogVisible.value); -const title = computed(() => { - if (!form.value) return t(`components.${ns}.dialog.run.title`); - return `${t(`components.${ns}.dialog.run.title`)} - ${form.value.name}`; -}); - const formRef = ref(); const onClose = () => { @@ -43,6 +38,7 @@ defineOptions({ name: 'ClCreateTaskDialog' }); :title="t('components.task.dialog.create.title')" :visible="visible" class-name="run-spider-dialog" + width="1024px" @close="onClose" @confirm="onConfirm" > diff --git a/src/i18n/lang/en/components/spider.ts b/src/i18n/lang/en/components/spider.ts index 0708fc124a452..da4edb49be34b 100644 --- a/src/i18n/lang/en/components/spider.ts +++ b/src/i18n/lang/en/components/spider.ts @@ -21,6 +21,7 @@ const spider: LComponentsSpider = { startUrls: 'Start URLs', domains: 'Domains', }, + templateDoc: 'Template Documentation', }, actions: { files: { diff --git a/src/i18n/lang/zh/components/spider.ts b/src/i18n/lang/zh/components/spider.ts index 44c93046a34ea..2594cdf3e8e34 100644 --- a/src/i18n/lang/zh/components/spider.ts +++ b/src/i18n/lang/zh/components/spider.ts @@ -21,6 +21,7 @@ const spider: LComponentsSpider = { startUrls: '起始 URL', domains: '域名', }, + templateDoc: '模版相关文档', }, actions: { files: { diff --git a/src/interfaces/i18n/components/spider.d.ts b/src/interfaces/i18n/components/spider.d.ts index f69a662e8e1f8..aa6cb6ab49462 100644 --- a/src/interfaces/i18n/components/spider.d.ts +++ b/src/interfaces/i18n/components/spider.d.ts @@ -21,6 +21,7 @@ interface LComponentsSpider { startUrls: string; domains: string; }; + templateDoc: string; }; actions: { files: { diff --git a/src/interfaces/models/spider.d.ts b/src/interfaces/models/spider.d.ts index 52de5fc18b37d..f7d75f6ba1f3d 100644 --- a/src/interfaces/models/spider.d.ts +++ b/src/interfaces/models/spider.d.ts @@ -70,10 +70,20 @@ export declare global { domains?: string; } + interface SpiderTemplateGroup { + lang: DependencyLang; + label: string; + icon?: Icon; + templates: SpiderTemplate[]; + } + interface SpiderTemplate { name: SpiderTemplateName; label: string; + icon?: Icon; cmd: string; params?: SpiderTemplateParams; + doc_url?: string; + doc_label?: string; } } diff --git a/src/utils/spider.ts b/src/utils/spider.ts index 2c57d7d73cd18..0e713de3a8570 100644 --- a/src/utils/spider.ts +++ b/src/utils/spider.ts @@ -1,59 +1,126 @@ -export const getSpiderTemplates = (): SpiderTemplate[] => { +import { getI18n } from '@/i18n'; + +export const getSpiderTemplateGroups = (): SpiderTemplateGroup[] => { + const locale = getI18n().global.locale.value; return [ { - name: 'scrapy', - label: 'Scrapy', - cmd: 'scrapy crawl scrapy_spider', - params: { - spider_name: 'scrapy_spider', - start_urls: 'http://example.com', - domains: 'example.com', - }, - }, - { - name: 'bs4', - label: 'BeautifulSoup', - cmd: 'python main.py', - }, - { - name: 'selenium', - label: 'Selenium', - cmd: 'python main.py', - }, - { - name: 'puppeteer', - label: 'Puppeteer', - cmd: 'node main.js', - }, - { - name: 'playwright', - label: 'Playwright', - cmd: 'node main.js', - }, - { - name: 'colly', - label: 'Colly', - cmd: 'go run main.go', - }, - { - name: 'python', - label: 'Basic Python', - cmd: 'python main.py', + lang: 'python', + label: 'Python', + icon: ['fab', 'python'], + templates: [ + { + name: 'scrapy', + label: 'Scrapy', + cmd: 'scrapy crawl scrapy_spider', + icon: ['svg', 'scrapy'], + params: { + spider_name: 'scrapy_spider', + start_urls: 'http://example.com', + domains: 'example.com', + }, + doc_url: 'https://docs.scrapy.org/en/latest/intro/overview.html', + doc_label: 'Scrapy Documentation', + }, + { + name: 'bs4', + label: 'BeautifulSoup', + icon: ['fa', 'leaf'], + cmd: 'python main.py', + doc_url: `https://www.crummy.com/software/BeautifulSoup/bs4/doc/`, + doc_label: 'BeautifulSoup Documentation', + }, + { + name: 'selenium', + label: 'Selenium', + icon: ['svg', 'selenium'], + cmd: 'python main.py', + doc_url: `https://selenium-python.readthedocs.io/`, + doc_label: 'Selenium Documentation', + }, + { + name: 'python', + label: 'Basic Python', + icon: ['fab', 'python'], + cmd: 'python main.py', + doc_url: `https://www.python.org/`, + doc_label: 'Python Documentation', + }, + ], }, { - name: 'node', - label: 'Basic Node.js', - cmd: 'node main.js', + lang: 'node', + label: 'Node.js', + icon: ['fab', 'node-js'], + templates: [ + { + name: 'puppeteer', + label: 'Puppeteer', + icon: ['svg', 'puppeteer'], + cmd: 'node main.js', + doc_url: `https://pptr.dev/`, + doc_label: 'Puppeteer Documentation', + }, + { + name: 'playwright', + label: 'Playwright', + icon: ['svg', 'playwright'], + cmd: 'node main.js', + doc_url: `https://playwright.dev/`, + doc_label: 'Playwright Documentation', + }, + { + name: 'node', + label: 'Basic Node.js', + icon: ['fab', 'node-js'], + cmd: 'node main.js', + doc_url: `https://nodejs.org/${locale === 'zh' ? 'zh-cn' : 'en'}/`, + doc_label: 'Node.js Documentation', + }, + ], }, { - name: 'go', - label: 'Basic Go', - cmd: 'go run main.go', + lang: 'go', + label: 'Go', + icon: ['svg', 'go'], + templates: [ + { + name: 'colly', + label: 'Colly', + icon: ['svg', 'colly'], + cmd: 'go run main.go', + doc_url: `https://go-colly.org/`, + doc_label: 'Go Colly Documentation', + }, + { + name: 'go', + label: 'Basic Go', + icon: ['svg', 'go'], + cmd: 'go run main.go', + doc_url: `https://go.dev/`, + doc_label: 'Go Documentation', + }, + ], }, { - name: 'java', - label: 'Basic Java', - cmd: 'mvn clean compile exec:java', + lang: 'java', + label: 'Java', + templates: [ + { + name: 'java', + label: 'Basic Java', + icon: ['fab', 'java'], + cmd: 'mvn clean compile exec:java', + }, + ], }, ]; }; + +export const getSpiderTemplates = (): SpiderTemplate[] => { + return getSpiderTemplateGroups().reduce( + (acc: SpiderTemplate[], group: SpiderTemplateGroup) => { + return acc.concat(group.templates); + }, + [] + ); +}; diff --git a/src/views/dependency/list/DependencyList.vue b/src/views/dependency/list/DependencyList.vue index 78db10ed9d43e..02a4c58f312ea 100644 --- a/src/views/dependency/list/DependencyList.vue +++ b/src/views/dependency/list/DependencyList.vue @@ -150,4 +150,10 @@ defineOptions({ name: 'ClDependencyList' }); line-height: 1.2; } } + +.icon-wrapper { + &:deep(img) { + filter: grayscale(100); + } +}