最新消息:请随时分享你的乐趣!

logstash jdbc 配置文件

技术博客 磊子 313浏览 0评论

JDBC

[root@y1 conf]# more oracle.conf
input {
    jdbc {
      jdbc_connection_string => "jdbc:oracle:thin:@192.168.0.21:1521/XE"
      jdbc_user => "yjhh"
      jdbc_password => "yjhh"
      jdbc_driver_library => "/usr/share/logstash/conf/ojdbc7.jar"
      jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
      jdbc_paging_enabled => "true"
      jdbc_page_size => "50000"
      statement => "select * from EMP t"
      #statement_filepath => "/usr/share/logstash-5.6.1/config_oracle/jdbc.sql"
      #schedule => "* * * * *"
      type => "jdbc"
    }
}
output{
 elasticsearch{
      hosts  => "192.168.0.22:9200"
      index => "emp"
      document_type => "info"
      document_id => "%{empno}"
    }
}

JDBC ES父子节点配置

input {
    jdbc {
        jdbc_connection_string => "jdbc:oracle:thin:@//10.61.120.46:1521/orcl"
        jdbc_user => "SXYMBQ"
        jdbc_password => "SXYMBQ20162017"
        jdbc_driver_library => "/usr/share/logstash-5.6.1/ojdbc7.jar"
        jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
        statement => "SELECT
    a.RELOCATIONID,a.REMOVALTYPEID,a.OBJ_RELOCA_HZXM,a.OBJ_RELOCA_XB,a.OBJ_RELOCA_SFZH,a.OBJ_RELOCA_HJBH,a.OBJ_RELOCA_YHYDBH,a.OBJ_RELOCA_JTRK,a.OBJ_RELOCA_LXDH,a.OBJ_RELOCA_JTYZZ,a.OBJ_RELOCA_YZJDMJ,a.OBJ_RELOCA_YFWJZMJ,
    a.OBJ_RELOCA_YFWJG,a.OBJ_RELOCA_LDLRS,a.OBJ_RELOCA_JTZYSRLY,a.OBJ_RELOCA_PKSX,a.OBJ_RELOCA_ZYPKYY,a.OBJ_RELOCA_JZHJ,a.OBJ_RELOCA_JTNSR,a.OBJ_RELOCA_FWRJJZMJ,a.OBJ_RELOCA_BZSJ,a.PROVINCEID,a.CITYID,a.COUNTRYID,a.TOWNID,
    a.DEPTID,a.DEPTNAME,a.DEPTNO,a.DEPTCODE,a.JD,a.WD,a.IMPORTDATE,a.DATASOURCE,a.OBJ_RELOCA_TBZRR,a.OBJ_RELOCA_TBRLXDH,a.OBJ_RELOCA_TBSJ,a.OBJ_RELOCA_BQHLX,a.PROVINCENAME,a.CITYNAME,a.COUNTRYNAME,a.TOWNNAME,a.OBJ_RELOCA_WHCD,
    a.OBJ_RELOCA_JKZK,a.OBJ_RELOCA_LDNL,a.ISDELETE,a.LASTUPDATETIME,a.LASTUPDATEUSER,a.REMOVALYEAR,
    c.FAMILYMEMBERID,c.OBJ_RELFAMILY_NAME,c.OBJ_RELFAMILY_YHZGX,c.OBJ_RELFAMILY_XB,c.OBJ_RELFAMILY_WHCD,c.OBJ_RELFAMILY_SFZH,c.OBJ_RELFAMILY_CJZH,c.OBJ_RELFAMILY_CSRQ,c.OBJ_RELFAMILY_JKZK,c.OBJ_RELFAMILY_LDNL,
    c.PROVINCE,c.CITY,c.COUNTRY,c.TOWN,c.SORT,c.OBJ_RELFAMILY_WGZK,c.OBJ_RELFAMILY_ZXZK,
    d.SUP_PEOPLEID,d.OBJ_SUPPEOPLE_XM,d.OBJ_SUPPEOPLE_XB,d.OBJ_SUPPEOPLE_ZWJB,d.OBJ_SUPPEOPLE_SZDW,d.OBJ_SUPPEOPLE_KSSJ,d.OBJ_SUPPEOPLE_JSSJ,d.OBJ_SUPPEOPLE_DWLSGX,d.OBJ_SUPPEOPLE_LXDH,d.OBJ_SUPPEOPLE_DWDZ,
    d.OBJ_SUPPEOPLE_BFCS,
    e.MEASURESID,e.OBJ_MEASURES_CYFCCS,e.OBJ_MEASURES_HXTPCS,e.OBJ_MEASURES_BQHSRLY,e.OBJ_MEASURES_QTTPCS,e.OBJ_MEASURES_BQHJTJYRS,e.OBJ_MEASURES_BQHJTNSR,e.OBJ_MEASURES_SFTP,e.OBJ_MEASURES_TPXHSJ,
    e.OBJ_MEASURES_TBZRR,e.OBJ_MEASURES_TBRLXDH,e.OBJ_MEASURES_TBSJ,e.OBJ_MEASURES_STATE,
    f.MOVE_SUPPORTID,f.OBJ_MOVESUP_QRD,f.OBJ_MOVESUP_AZFS,f.OBJ_MOVESUP_AZDMC,f.OBJ_MOVESUP_AZDLX,f.OBJ_MOVESUP_FWLX,f.OBJ_MOVESUP_JSFS,f.OBJ_MOVESUP_JYSJ,f.OBJ_MOVESUP_SJRZSJ,f.OBJ_MOVESUP_AZFWJZMJ,
    f.OBJ_MOVESUP_JGFCB,f.OBJ_MOVESUP_JGFBZZJ,f.OBJ_MOVESUP_ZCJE,f.OBJ_MOVESUP_SSSJRWND,f.OBJ_MOVESUP_SSGJRWND,f.OBJ_MOVESUP_TTJZJDSJ,f.OBJ_MOVESUP_TTJZJDMJ,f.OBJ_MOVESUP_TTJZJJHBZZJ,f.OBJ_MOVESUP_XFHYZP,
    f.OBJ_MOVESUP_TBZRR,f.OBJ_MOVESUP_TBRLXDH,f.OBJ_MOVESUP_TBSJ,f.OBJ_MOVESUP_STATE,f.OBJ_MOVESUP_RJJZMJ,f.OBJ_MOVESUP_AZDID
    ,f.POINTX,f.POINTY,f.DEPTID_OLD,f.DEPTNAME_OLD,f.DEPTNO_OLD,f.DEPTCODE_OLD,f.OBJ_MOVESUP_JZAZFS,
    b.removaltypename
FROM
    obj_reloca_info a
    LEFT JOIN obj_removaltype_info b ON a.removaltypeid = b.removaltypeid
    LEFT JOIN obj_family_inf c ON a.relocationid = c.relocationid
    LEFT JOIN obj_suppeople_info d ON a.relocationid = d.relocationid
    LEFT JOIN obj_measures_info e ON a.relocationid = e.relocationid
    LEFT JOIN obj_movesup_inf f ON a.relocationid = f.relocationid"
        jdbc_paging_enabled => "true"
        jdbc_page_size => "10000"
        #schedule => "* * * * *"
        #type => "jdbc"
    }
}
filter {
     aggregate {
       task_id => "%{relocationid}"
       code => "
        map['relocationid'] = event.get('relocationid')
        map['obj_reloca_hzxm'] = event.get('obj_reloca_hzxm')
        map['obj_reloca_xb'] = event.get('obj_reloca_xb')
        map['obj_reloca_sfzh'] = event.get('obj_reloca_sfzh')
        map['obj_reloca_hjbh'] = event.get('obj_reloca_hjbh')
        map['obj_reloca_yhydbh'] = event.get('obj_reloca_yhydbh')
        map['obj_reloca_jtrk'] = event.get('obj_reloca_jtrk')
        map['obj_reloca_lxdh'] = event.get('obj_reloca_lxdh')
        map['obj_reloca_jtyzz'] = event.get('obj_reloca_jtyzz')
        map['obj_reloca_yzjdmj'] = event.get('obj_reloca_yzjdmj')
        map['removaltypename'] = event.get('removaltypename')
        map['objfamilyinf'] ||= []
        map['objfamilyinf'] << {
                'relocationid' => event.get('relocationid'),
                'familymemberid' => event.get('familymemberid'),
                'obj_relfamily_name' => event.get('obj_relfamily_name'),
                'obj_relfamily_yhzgx' => event.get('obj_relfamily_yhzgx'),
                'obj_relfamily_xb' => event.get('obj_relfamily_xb'),
                'obj_relfamily_whcd' => event.get('obj_relfamily_whcd'),
                'obj_relfamily_sfzh' => event.get('obj_relfamily_sfzh'),
                'obj_relfamily_cjzh' => event.get('obj_relfamily_cjzh'),
                'obj_relfamily_csrq' => event.get('obj_relfamily_csrq'),
                'obj_relfamily_jkzk' => event.get('obj_relfamily_jkzk'),
                'obj_relfamily_ldnl' => event.get('obj_relfamily_ldnl'),
                'province' => event.get('province'),
                'city' => event.get('city'),
                'country' => event.get('country'),
                'town' => event.get('town'),
                'deptid' => event.get('deptid'),
                'deptname' => event.get('deptname'),
                'deptno' => event.get('deptno'),
                'deptcode' => event.get('deptcode'),
                'removaltypeid' => event.get('removaltypeid'),
                'sort' => event.get('sort'),
                'obj_relfamily_wgzk' => event.get('obj_relfamily_wgzk'),
                'obj_relfamily_zxzk' => event.get('obj_relfamily_zxzk'),
                'isdelete' => event.get('isdelete')
        }
        map['objsuppeopleinfo'] ||= {
                'sup_peopleid' => event.get('sup_peopleid'),
                'relocationid' => event.get('relocationid'),
                'obj_suppeople_xm' => event.get('obj_suppeople_xm'),
                'obj_suppeople_xb' => event.get('obj_suppeople_xb'),
                'obj_suppeople_zwjb' => event.get('obj_suppeople_zwjb'),
                'obj_suppeople_szdw' => event.get('obj_suppeople_szdw'),
                'obj_suppeople_kssj' => event.get('obj_suppeople_kssj'),
                'obj_suppeople_jssj' => event.get('obj_suppeople_jssj'),
                'obj_suppeople_dwlsgx' => event.get('obj_suppeople_dwlsgx'),
                'obj_suppeople_lxdh' => event.get('obj_suppeople_lxdh'),
                'obj_suppeople_dwdz' => event.get('obj_suppeople_dwdz'),
                'obj_suppeople_bfcs' => event.get('obj_suppeople_bfcs')
        }
        map['objmeasuresinfo'] ||= {
                'measuresid' => event.get('measuresid'),
                'relocationid' => event.get('relocationid'),
                'obj_measures_cyfccs' => event.get('obj_measures_cyfccs'),
                'obj_measures_hxtpcs' => event.get('obj_measures_hxtpcs')
        }
        map['objmovesupinf'] ||= {
                'move_supportid' => event.get('move_supportid'),
                'relocationid' => event.get('relocationid'),
                'obj_movesup_qrd' => event.get('obj_movesup_qrd'),
                'obj_movesup_azfs' => event.get('obj_movesup_azfs')
        }
    map['familyMemberCount'] ||= 0
    map['familyMemberCount'] += 1
        event.cancel()
        "
       push_previous_map_as_event => true
       timeout => 120
     }
}
output {
    elasticsearch {
        hosts => "10.61.120.24:9200"
        user => "elastic"
        password => "changeme"
        #protocol => http
        index => "obj_reloca_info"
        document_type => "reloca_info"
        document_id => "%{relocationid}"
    }
}

转载请注明:印迹. » logstash jdbc 配置文件

发表我的评论
取消评论

表情