尝试在弹性中启用搜索建议字段时，从 sql server 导入 Logstash 时遇到困难

Question

使用 sql server 表作为我的基础，其中包含列 Id (nvarchar)、标题 (nvarchar)、教师 (nvarchar)、网站类别 (nvarchar)；我打算利用 Elasticsearch 映射和一个名为 suggest 的字段来实现专门针对列标题和教师的搜索建议功能。 我确定错误原因的位置是 Logstash .conf 文件输出部分中的脚本和 scipt_params json 对象，语法可能不正确，但不知道如何修复它！感谢每一个提示和帮助。 我保证所有路由都可用，例如 template.json 和证书路径等。 cpu_arch:X86_64 ，操作系统：ubuntu 22.04.3 LTS jellyfish CN，ELK_ver 安装在我的本地机器上：8.10.2

下面是Logstash_配置文件：

input {
 jdbc {
    jdbc_driver_library => "/home/erfan/SearchHerkunft/logstash-8.10.2/lib/mssql-jdbc-12.4.1.jre11.jar"
    jdbc_driver_class => "com.microsoft.sqlserver.jdbc.SQLServerDriver"
    jdbc_connection_string => "jdbc:sqlserver://localhost:1433;database=WebScrapingProject_DB;encrypt=true;trustServerCertificate=true;trustStore=/home/erfan/SearchHerkunft/logstash-8.10.2/config/http_ca.crt"
    jdbc_user => "sa"
    jdbc_password => "Warner123#"
    jdbc_paging_enabled => true
    jdbc_fetch_size => 10
    schedule => "* * * * *"
    statement => "SELECT TOP (1000) [Id]
      ,[Title]
      ,[Teacher]
      ,[WebSiteCategory]
 FROM [WebScrapingProject_DB].[dbo].[Test]"
 }
}

filter {
 mutate {
    copy => { "Id" => "[@metadata][_id]" }
    remove_field => ["Id", "@version"]
 }
 mutate {
    convert => { "[@metadata][_id]" => "string" }
 }
}

output {
 elasticsearch {
    hosts => ["https://localhost:9200"]
    ssl => true
    ssl_certificate_verification => true
    cacert => '/home/erfan/SearchHerkunft/logstash-8.10.2/config/http_ca.crt'
    user => "elastic"
    password => "AE1Qj2*vb73VIkpp4Nyh"
    index => "yabland"
    document_id => "%{[@metadata][_id]}"
    document_type => "_doc"
    routing => "%{[@metadata][_id]}"
    action => "update"
    doc_as_upsert => true
    script_params => {
 "title" => "%{[@metadata][Title]}",
 "teacher" => "%{[@metadata][Teacher]}"
},
    script => "ctx._source.Suggest = [ 'input': [params.title, params.teacher], 'weight': 10]"
    template => '/home/erfan/SearchHerkunft/logstash-8.10.2/config/template.json'
 }
}

以及输出部分中提到的template.json：

{
  "index_patterns": ["yabland-*"],
  "mappings": {
    "properties": {
      "id": { "type": "keyword" },
      "title": { "type": "text" },
      "teacher": { "type": "text" },
      "webSiteCategory": { "type": "text" },
      "suggest": { "type": "completion", "analyzer": "simple" }
    }
  }
}

此外，您将看到用于执行 Logstash 的终端命令，引用 .conf 文件，会导致终端日志错误，如下所示：

erfan@erfan-SVF14218SAW:~/SearchHerkunft/logstash-8.10.2$ bin/logstash -f ~/SearchHerkunft/logstash-8.10.2/config/sqlJDBC.conf

[ERROR] 2023-10-17 12:00:00.318 [Converge PipelineAction::Create<main>] agent - Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of [ \\t\\r\\n], \"#\", \"{\", \"}\" at line 44, column 36 (byte 1360) after output {\n elasticsearch {\n    hosts => [\"https://localhost:9200\"]\n    ssl => true\n    ssl_certificate_verification => true\n    cacert => '/home/erfan/SearchHerkunft/logstash-8.10.2/config/http_ca.crt'\n    user => \"elastic\"\n    password => \"AE1Qj2*vb73VIkpp4Nyh\"\n    index => \"yabland\"\n    document_id => \"%{[@metadata][_id]}\"\n    document_type => \"_doc\"\n    routing => \"%{[@metadata][_id]}\"\n    action => \"update\"\n    doc_as_upsert => true\n    script_params => {\n \"title\" => \"%{[@metadata][Title]}\"", :backtrace=>["/home/erfan/SearchHerkunft/logstash-8.10.2/logstash-core/lib/logstash/compiler.rb:32:in `compile_imperative'", "org/logstash/execution/AbstractPipelineExt.java:239:in `initialize'", "org/logstash/execution/AbstractPipelineExt.java:173:in `initialize'", "/home/erfan/SearchHerkunft/logstash-8.10.2/logstash-core/lib/logstash/java_pipeline.rb:48:in `initialize'", "org/jruby/RubyClass.java:931:in `new'", "/home/erfan/SearchHerkunft/logstash-8.10.2/logstash-core/lib/logstash/pipeline_action/create.rb:49:in `execute'", "/home/erfan/SearchHerkunft/logstash-8.10.2/logstash-core/lib/logstash/agent.rb:386:in `block in converge_state'"]}
[INFO ] 2023-10-17 12:00:00.398 [LogStash::Runner] runner - Logstash shut down.

预计运行并开始通过管道进入 Elasticsearch 服务

Answer 1

有两个问题。

第一个问题是在你的映射中，

suggest

都是小写的

  "suggest": { "type": "completion", "analyzer": "simple" }
   ^
   |

并且在您的脚本中，该字段的首字母大写

ctx._source.Suggest
            ^
            |
       change this

其次，您收到的错误是因为elasticsearch 输出中没有

script_params

设置。您需要使用 params.event.get('field') 语法访问

事件字段

。所以你的脚本需要看起来像这样：

script => "ctx._source.suggest = [ 'input': [params.event.get('title'), params.event.get('teacher')], 'weight': 10]"