From 43d648dcd3e71acb2464ab6613056f05300845a4 Mon Sep 17 00:00:00 2001 From: Li ZongYing Date: Fri, 17 Nov 2023 15:58:21 +0800 Subject: [PATCH 1/3] fix nonepipeline config --- README.md | 2 ++ README_CN.md | 2 ++ docs/content/en/docs/usage/tools/_index.md | 2 ++ docs/content/zh/docs/usage/tools/_index.md | 2 ++ pkg/config/config.go | 2 +- pkg/exporter/exporter.go | 9 ++------- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index eabdc47..6fb3556 100644 --- a/README.md +++ b/README.md @@ -1211,6 +1211,8 @@ For more examples, you can refer to the following project. * `-s` Self-signed server certificate. If not set, the default CA certificate of this project will be used for signing. * `-c` Create a new CA certificate. If not set, the default CA certificate of this project will be used. +* `-i` Add server IP addresses, separated by commas. +* `-h` Add server domain names, separated by commas. dev diff --git a/README_CN.md b/README_CN.md index d259644..0f4dea4 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1067,6 +1067,8 @@ go run example_spider.go -c example.yml -n example -f TestOk -m once * -s 自签名服务器证书。如果不设置,会使用本项目默认ca证书进行签名 * -c 新创建ca证书。如果不设置,会使用本项目默认ca证书 +* -i 添加服务器ip,逗号分割 +* -h 添加服务器域名,逗号分割 开发 diff --git a/docs/content/en/docs/usage/tools/_index.md b/docs/content/en/docs/usage/tools/_index.md index 9e45c59..76aaba6 100644 --- a/docs/content/en/docs/usage/tools/_index.md +++ b/docs/content/en/docs/usage/tools/_index.md @@ -9,6 +9,8 @@ title: "Tools" * `-s` Self-signed server certificate. If not set, the default CA certificate of this project will be used for signing. * `-c` Create a new CA certificate. If not set, the default CA certificate of this project will be used. +* `-i` Add server IP addresses, separated by commas. +* `-h` Add server domain names, separated by commas. dev diff --git a/docs/content/zh/docs/usage/tools/_index.md b/docs/content/zh/docs/usage/tools/_index.md index ab17dfe..b423449 100644 --- a/docs/content/zh/docs/usage/tools/_index.md +++ b/docs/content/zh/docs/usage/tools/_index.md @@ -9,6 +9,8 @@ title: "工具" * -s 自签名服务器证书。如果不设置,会使用本项目默认ca证书进行签名 * -c 新创建ca证书。如果不设置,会使用本项目默认ca证书 +* -i 添加服务器ip,逗号分割 +* -h 添加服务器域名,逗号分割 开发 diff --git a/pkg/config/config.go b/pkg/config/config.go index 8d719f4..9ed878c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -430,7 +430,7 @@ func (c *Config) GetEnableNonePipeline() bool { c.EnableCsvPipeline = &enableNonePipeline } - return *c.EnableCsvPipeline + return *c.EnableNonePipeline } func (c *Config) GetEnableCsvPipeline() bool { if c.EnableCsvPipeline == nil { diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go index 3f87db9..44de73d 100644 --- a/pkg/exporter/exporter.go +++ b/pkg/exporter/exporter.go @@ -1,7 +1,6 @@ package exporter import ( - "errors" "github.com/lizongying/go-crawler/pkg" "github.com/lizongying/go-crawler/pkg/pipelines" "reflect" @@ -19,14 +18,10 @@ type Exporter struct { func (e *Exporter) Export(item pkg.Item) (err error) { for _, v := range e.pipelines { - e := v.ProcessItem(item) - if e != nil { - err = errors.Join(err, e) + if err = v.ProcessItem(item); err != nil { + e.logger.Error(err) } } - if err != nil { - e.logger.Error(err) - } return } From 3673567ecc8f83e7691af7e1ce263216851a2297 Mon Sep 17 00:00:00 2001 From: Li ZongYing Date: Fri, 17 Nov 2023 19:58:15 +0800 Subject: [PATCH 2/3] admin support search --- web/ui/src/views/HomeView.vue | 12 +- web/ui/src/views/JobsView.vue | 330 +++++++++++++++---------- web/ui/src/views/NodesView.vue | 370 +++++++++++++++++----------- web/ui/src/views/RecordsView.vue | 218 ++++++++++++----- web/ui/src/views/SpidersView.vue | 402 +++++++++++++++++++------------ web/ui/src/views/TasksView.vue | 296 +++++++++++++++-------- 6 files changed, 1035 insertions(+), 593 deletions(-) diff --git a/web/ui/src/views/HomeView.vue b/web/ui/src/views/HomeView.vue index 0d5e316..d3b9c11 100644 --- a/web/ui/src/views/HomeView.vue +++ b/web/ui/src/views/HomeView.vue @@ -40,13 +40,17 @@ import {onBeforeUnmount, ref} from "vue"; const nodesStore = useNodesStore() nodesStore.GetNodes() +// auto refresh +const checked1 = ref(true) +const checked1Disable = ref(true) +let interval = 0 const refresh = () => { nodesStore.GetNodes() } -const checked1 = ref(false) -const checked1Disable = ref(false) - -let interval = null +refresh() +if (checked1.value) { + interval = setInterval(refresh, 1000) +} const changeSwitch = () => { if (checked1.value) { interval = setInterval(refresh, 1000) diff --git a/web/ui/src/views/JobsView.vue b/web/ui/src/views/JobsView.vue index 4212e70..ed72df8 100644 --- a/web/ui/src/views/JobsView.vue +++ b/web/ui/src/views/JobsView.vue @@ -17,8 +17,38 @@ - - diff --git a/web/ui/src/views/NodesView.vue b/web/ui/src/views/NodesView.vue index a9e5403..57f8ef0 100644 --- a/web/ui/src/views/NodesView.vue +++ b/web/ui/src/views/NodesView.vue @@ -16,8 +16,38 @@ - - diff --git a/web/ui/src/views/RecordsView.vue b/web/ui/src/views/RecordsView.vue index bd5d33b..95de045 100644 --- a/web/ui/src/views/RecordsView.vue +++ b/web/ui/src/views/RecordsView.vue @@ -16,8 +16,38 @@ - - diff --git a/web/ui/src/views/SpidersView.vue b/web/ui/src/views/SpidersView.vue index 21f49e2..9ee640d 100644 --- a/web/ui/src/views/SpidersView.vue +++ b/web/ui/src/views/SpidersView.vue @@ -17,8 +17,38 @@ - - diff --git a/web/ui/src/views/TasksView.vue b/web/ui/src/views/TasksView.vue index 2fc988c..1034b88 100644 --- a/web/ui/src/views/TasksView.vue +++ b/web/ui/src/views/TasksView.vue @@ -16,8 +16,38 @@ - - From cc60ec06fa690ee085a8b1c6681613b75733298d Mon Sep 17 00:00:00 2001 From: Li ZongYing Date: Fri, 17 Nov 2023 20:38:10 +0800 Subject: [PATCH 3/3] fix admin filter --- web/ui/src/views/JobsView.vue | 5 +++++ web/ui/src/views/NodesView.vue | 4 ++++ web/ui/src/views/RecordsView.vue | 4 ++++ web/ui/src/views/SpidersView.vue | 4 ++++ web/ui/src/views/TasksView.vue | 4 ++++ 5 files changed, 21 insertions(+) diff --git a/web/ui/src/views/JobsView.vue b/web/ui/src/views/JobsView.vue index ed72df8..757802a 100644 --- a/web/ui/src/views/JobsView.vue +++ b/web/ui/src/views/JobsView.vue @@ -253,6 +253,7 @@ const {query} = useRoute(); if ('id' in query) { filteredInfo.id = [query.id] } + const columns = computed(() => { return [ { @@ -565,11 +566,15 @@ const state = reactive({ }); const searchInput = ref(); const handleSearch = (selectedKeys, confirm, dataIndex) => { + filteredInfo[dataIndex] = selectedKeys confirm(); state.searchText = selectedKeys[0]; state.searchedColumn = dataIndex; }; const handleReset = clearFilters => { + Object.keys(filteredInfo).forEach(key => { + delete filteredInfo[key]; + }); clearFilters({ confirm: true, }); diff --git a/web/ui/src/views/NodesView.vue b/web/ui/src/views/NodesView.vue index 57f8ef0..c152e49 100644 --- a/web/ui/src/views/NodesView.vue +++ b/web/ui/src/views/NodesView.vue @@ -366,11 +366,15 @@ const state = reactive({ }); const searchInput = ref(); const handleSearch = (selectedKeys, confirm, dataIndex) => { + filteredInfo[dataIndex] = selectedKeys confirm(); state.searchText = selectedKeys[0]; state.searchedColumn = dataIndex; }; const handleReset = clearFilters => { + Object.keys(filteredInfo).forEach(key => { + delete filteredInfo[key]; + }); clearFilters({ confirm: true, }); diff --git a/web/ui/src/views/RecordsView.vue b/web/ui/src/views/RecordsView.vue index 95de045..9a2e3d2 100644 --- a/web/ui/src/views/RecordsView.vue +++ b/web/ui/src/views/RecordsView.vue @@ -234,11 +234,15 @@ const state = reactive({ }); const searchInput = ref(); const handleSearch = (selectedKeys, confirm, dataIndex) => { + filteredInfo[dataIndex] = selectedKeys confirm(); state.searchText = selectedKeys[0]; state.searchedColumn = dataIndex; }; const handleReset = clearFilters => { + Object.keys(filteredInfo).forEach(key => { + delete filteredInfo[key]; + }); clearFilters({ confirm: true, }); diff --git a/web/ui/src/views/SpidersView.vue b/web/ui/src/views/SpidersView.vue index 9ee640d..e4c5f63 100644 --- a/web/ui/src/views/SpidersView.vue +++ b/web/ui/src/views/SpidersView.vue @@ -424,11 +424,15 @@ const state = reactive({ }); const searchInput = ref(); const handleSearch = (selectedKeys, confirm, dataIndex) => { + filteredInfo[dataIndex] = selectedKeys confirm(); state.searchText = selectedKeys[0]; state.searchedColumn = dataIndex; }; const handleReset = clearFilters => { + Object.keys(filteredInfo).forEach(key => { + delete filteredInfo[key]; + }); clearFilters({ confirm: true, }); diff --git a/web/ui/src/views/TasksView.vue b/web/ui/src/views/TasksView.vue index 1034b88..4cc6a8b 100644 --- a/web/ui/src/views/TasksView.vue +++ b/web/ui/src/views/TasksView.vue @@ -303,11 +303,15 @@ const state = reactive({ }); const searchInput = ref(); const handleSearch = (selectedKeys, confirm, dataIndex) => { + filteredInfo[dataIndex] = selectedKeys confirm(); state.searchText = selectedKeys[0]; state.searchedColumn = dataIndex; }; const handleReset = clearFilters => { + Object.keys(filteredInfo).forEach(key => { + delete filteredInfo[key]; + }); clearFilters({ confirm: true, });