Skip to content

M
mysql2es
提交 0dd6348a authored 作者: Jack Jie's avatar Jack Jie
浏览文件
操作

Initial commit

上级
隐藏空白字符变更
内嵌 并排
正在显示 包含 554 行增加 和 0 行删除
.idea/.gitignore 0 → 100644
浏览文件 @ 0dd6348a
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
.idea/modules.xml 0 → 100644
浏览文件 @ 0dd6348a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/pg2es-test.iml" filepath="$PROJECT_DIR$/.idea/pg2es-test.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/pg2es-test.iml 0 → 100644
浏览文件 @ 0dd6348a
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
config.yaml 0 → 100644
浏览文件 @ 0dd6348a
mysql:
host: "192.168.60.10"
user: "root"
password: "gtmap"
dbname: "gtc_db"
port: "3306"
timezone: "Asia/Shanghai"
charset: "UTF8"
postgres:
host: "192.168.2.54"
user: "system"
password: "gtis"
dbname: "test"
port: "54321"
sslmode: "disable"
timezone: "Asia/Shanghai"
charset: "UTF8"
schemaname: "public"
elasticsearch:
url: "http://192.168.2.194:9200"
index: "scp-platform-"
query:
start_time: "2024-06-21 00:00:00"
end_time: "2024-12-01 00:00:00"
go.mod 0 → 100644
浏览文件 @ 0dd6348a
module mysql2es-test
go 1.22
require (
github.com/google/uuid v1.6.0
github.com/olivere/elastic/v7 v7.0.32
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/mysql v1.5.7
gorm.io/driver/postgres v1.5.7
gorm.io/gorm v1.25.11
)
require (
github.com/go-sql-driver/mysql v1.7.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.4.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/text v0.18.0 // indirect
)
go.sum 0 → 100644
浏览文件 @ 0dd6348a
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY=
github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM=
gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
main.go 0 → 100644
浏览文件 @ 0dd6348a
package main
import (
"context"
"encoding/json"
"fmt"
"github.com/google/uuid"
"github.com/olivere/elastic/v7"
"gopkg.in/yaml.v3"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"log"
"os"
"strconv"
"strings"
"time"
)
// Config represents the structure of the configuration file
type Config struct {
Postgres PostgresConfig `json:"postgres" yaml:"postgres"`
MySQL MySQLConfig `json:"mysql" yaml:"mysql"`
Elasticsearch ElasticsearchConfig `json:"elasticsearch" yaml:"elasticsearch"`
Query QueryConfig `json:"query" yaml:"query"`
}
type MySQLConfig struct {
Host string `json:"host" yaml:"host"`
User string `json:"user" yaml:"user"`
Password string `json:"password" yaml:"password"`
DBName string `json:"dbname" yaml:"dbname"`
Port string `json:"port" yaml:"port"`
Charset string `json:"charset" yaml:"charset"`
}
type PostgresConfig struct {
Host string `json:"host" yaml:"host"`
User string `json:"user" yaml:"user"`
Password string `json:"password" yaml:"password"`
DBName string `json:"dbname" yaml:"dbname"`
Port string `json:"port" yaml:"port"`
SSLMode string `json:"sslmode" yaml:"sslmode"`
TimeZone string `json:"timezone" yaml:"timezone"`
Charset string `json:"charset" yaml:"charset"`
SchemaName string `json:"schemaname" yaml:"schemaname"`
}
type ElasticsearchConfig struct {
URL string `json:"url" yaml:"url"`
Index string `json:"index" yaml:"index"`
}
type QueryConfig struct {
StartTime string `json:"start_time" yaml:"start_time"`
EndTime string `json:"end_time" yaml:"end_time"`
}
// HistoryLog represents the structure of the history_log table
type HistoryLog struct {
Id string `gorm:"primaryKey;type:varchar(32);comment:主键，来源于 jk_history_business_service_api 表的 id 字段" json:"id"` // 主键
IdSysLog string `gorm:"type:varchar(32);comment:来源于 sys_log 表的 id 字段，用于保留 sys_log 表中的 id 值" json:"id_sys_log"` // 来源于 sys_log 表的 id
LogType int `gorm:"type:int;comment:日志类型（1登录日志，2操作日志），来源于 sys_log 表的 log_type 字段" json:"log_type"` // 日志类型
LogContent string `gorm:"type:varchar(1000);comment:日志内容，来源于 sys_log 表的 log_content 字段" json:"log_content"` // 日志内容
OperateType int `gorm:"type:int;comment:操作类型，来源于 sys_log 表的 operate_type 字段" json:"operate_type"` // 操作类型
Userid string `gorm:"type:varchar(32);comment:操作用户账号，来源于 sys_log 表的 userid 字段" json:"userid"` // 操作用户账号
Username string `gorm:"type:varchar(100);comment:操作用户名称，来源于 sys_log 表的 username 字段" json:"username"` // 操作用户名称
RemoteIp string `gorm:"type:varchar(100);comment:IP地址，来源于 sys_log 表的 ip 字段" json:"remote_ip"` // IP地址
Method string `gorm:"type:varchar(500);comment:请求的 Java 方法，来源于 sys_log 表的 method 字段" json:"method"` // 请求的 Java 方法
RequestUrl string `gorm:"type:varchar(255);comment:请求路径，来源于 sys_log 表的 request_url 字段" json:"request_url"` // 请求路径
InParams string `gorm:"type:text;comment:请求参数，来源于 sys_log 表的 request_param 字段" json:"in_params"` // 请求参数
RequestType string `gorm:"type:varchar(10);comment:请求类型，来源于 sys_log 表的 request_type 字段" json:"request_type"` // 请求类型
Duration int64 `gorm:"type:int;comment:耗时，来源于 sys_log 表的 cost_time 字段" json:"cost_time"` // 耗时
CreateBySyslog string `gorm:"type:varchar(32);comment:创建人，来源于 sys_log 表的 create_by 字段" json:"create_by_syslog"` // 创建人
CreateTimeSyslog time.Time `gorm:"type:timestamp;comment:创建时间，来源于 sys_log 表的 create_time 字段" json:"create_time_syslog"` // 创建时间
UpdateBySyslog string `gorm:"type:varchar(32);comment:更新人，来源于 sys_log 表的 update_by 字段" json:"update_by_syslog"` // 更新人
UpdateTimeSyslog time.Time `gorm:"type:timestamp;comment:更新时间，来源于 sys_log 表的 update_time 字段" json:"update_time_syslog"` // 更新时间
OutParams string `gorm:"type:text;comment:响应参数，来源于 sys_log 表的 resp_param 字段" json:"out_params"` // 响应参数
HistoryId string `gorm:"type:varchar(100);comment:关联的 jk_history_business_service_api 表主键 id，来源于 sys_log 表的 history_id 字段" json:"history_id"` // 关联的 jk_history_business_service_api 表主键 id
// 从 jk_history_business_service_api 表继承的字段
ServiceId string `gorm:"type:varchar(64);not null;default:'';comment:场景服务ID，来源于 jk_history_business_service_api 表的 scene_service_id 字段" json:"service_id"` // 场景服务ID
Url string `gorm:"type:varchar(250);comment:节点ID，来源于 jk_history_business_service_api 表的 node_id 字段" json:"url"` // 节点ID
ServiceCode string `gorm:"type:varchar(64);comment:业务服务CODE，来源于 jk_history_business_service_api 表的 business_service_api_code 字段" json:"service_code"` // 业务服务CODE
State string `gorm:"type:varchar(20);not null;default:'';comment:调用状态，来源于 jk_history_business_service_api 表的 status 字段" json:"state"` // 调用状态
AccessUuid string `gorm:"type:varchar(70);comment:应用APP key，来源于 jk_history_business_service_api 表的 app_key 字段" json:"access_uuid"` // 应用APP key
RegionCode string `gorm:"type:varchar(45);not null;default:'';comment:区域CODE(省或市或区)，来源于 jk_history_business_service_api 表的 region_code 字段" json:"region_code"` // 区域CODE(省或市或区)
RegionCodeCity string `gorm:"type:varchar(45);not null;default:'';comment:区域CODE(市)，来源于 jk_history_business_service_api 表的 region_code_city 字段" json:"region_code_city"` // 区域CODE(市)
Kind string `gorm:"type:varchar(20);not null;default:'';comment:类型，来源于 jk_history_business_service_api 表的 kind 字段" json:"kind"` // 类型
Uuid string `gorm:"type:varchar(500);comment:唯一标识符，来源于 jk_history_business_service_api 表的 uuid 字段" json:"uuid"` // 唯一标识符
Cost int `gorm:"type:int;comment:耗时，来源于 jk_history_business_service_api 表的 cost 字段" json:"duration"` // 耗时
RecordDay time.Time `gorm:"type:date;comment:记录日期，来源于 jk_history_business_service_api 表的 record_day 字段" json:"record_day"` // 记录日期
ThemeType string `gorm:"type:varchar(45);not null;default:'';comment:场景服务类型，来源于 jk_history_business_service_api 表的 service_type 字段" json:"theme_type"` // 场景服务类型
AccessXzqhdm string `gorm:"type:varchar(45);comment:外部传递的区县code，来源于 jk_history_business_service_api 表的 area_code 字段" json:"access_xzqhdm"` // 外部传递的区县code
GroupId string `gorm:"type:varchar(64);comment:分组ID，来源于 jk_history_business_service_api 表的 group_id 字段" json:"group_id"` // 分组ID
ResponseStatusCode int `gorm:"type:int;comment:接口响应状态码，来源于 jk_history_business_service_api 表的 status_code 字段" json:"response_status_code"` // 接口响应状态码
}
// ESLog represents the Elasticsearch log structure
type ESLog struct {
ID string `json:"id"`
TenantID string `json:"tenant_id"`
Name string `json:"name"`
Method string `json:"method"`
ParentID string `json:"parent_id"`
TraceID string `json:"trace_id"`
Event string `json:"event"`
Duration int `json:"duration"`
TimestampMillis string `json:"timestamp_millis"`
Timestamp int64 `json:"timestamp"`
Tags map[string]interface{} `json:"tags"`
Q []string `json:"_q"`
}
// LoadConfig loads the configuration from a file
func LoadConfig(filename string) (*Config, error) {
data, err := os.ReadFile(filename)
if err != nil {
return nil, err
}
var config Config
if ext := GetFileExtension(filename); ext == ".json" {
err = json.Unmarshal(data, &config)
} else if ext == ".yaml" || ext == ".yml" {
err = yaml.Unmarshal(data, &config)
}
return &config, err
}
// GetFileExtension returns the file extension
func GetFileExtension(filename string) string {
return filename[len(filename)-5:]
}
func main() {
// 加载配置文件
configFile := "config.yaml"
config, err := LoadConfig(configFile)
if err != nil {
log.Fatalf("failed to load config: %v", err)
}
// 配置 PostgreSQL 连接信息
postgres_dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=%s TimeZone=%s client_encoding=%s",
config.Postgres.Host, config.Postgres.User, config.Postgres.Password, config.Postgres.DBName, config.Postgres.Port, config.Postgres.SSLMode, config.Postgres.TimeZone, config.Postgres.Charset)
postgres_db, err := gorm.Open(postgres.New(postgres.Config{
DSN: postgres_dsn,
PreferSimpleProtocol: false, // disables implicit prepared statement usage
}), &gorm.Config{})
if err != nil {
log.Fatalf("failed to connect database: %v", err)
}
// 获取服务相关信息
serviceMap, err := GetServiceThemeInfo(config.Postgres.SchemaName, postgres_db)
if err != nil {
log.Fatalf("failed to get service theme info: %v", err)
}
// 获取系统名称
systemMap, err := GetServiceAccessInfo(postgres_db)
if err != nil {
log.Fatalf("failed to get service access builder info: %v", err)
}
// 配置 MySQL 连接信息
mysql_dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=%s&parseTime=True&loc=Local",
config.MySQL.User, config.MySQL.Password, config.MySQL.Host, config.MySQL.Port, config.MySQL.DBName, config.MySQL.Charset)
mysql_db, err := gorm.Open(mysql.Open(mysql_dsn), &gorm.Config{})
if err != nil {
log.Fatalf("failed to connect to MySQL: %v", err)
}
// 查询时间配置
startTime := config.Query.StartTime
endTime := config.Query.EndTime
// 查询 MySQL 中的数据
var logs []HistoryLog
if err := mysql_db.Model(HistoryLog{}).Table("history_log_view").
Where("create_time_syslog BETWEEN ? AND ?", startTime, endTime).Find(&logs).Error; err != nil {
log.Fatalf("failed to query logs: %v", err)
}
// Elasticsearch 连接
esClient, err := elastic.NewClient(elastic.SetURL(config.Elasticsearch.URL), elastic.SetSniff(false))
if err != nil {
log.Fatalf("failed to create Elasticsearch client: %v", err)
}
// 统计写入和失败的数量
totalCount := len(logs)
successCount := 0
failedIDs := []string{}
// 处理每条日志并写入 Elasticsearch
for _, logItem := range logs {
uuidStr := uuid.New().String()
fmt.Println(uuidStr)
instanceId := strings.ReplaceAll(uuid.New().String(), "-", "")
// 构造 Elasticsearch 日志结构
esLog := ESLog{
ID: uuidStr, // 日志主键
TenantID: "<nil>", // 租户ID
Name: "0.1.0", // 固定版本号
Event: "scp-default", // 事件
Method: "ProxyService.Redirect", // 请求方法
ParentID: "no-biz-id", // 父ID
TraceID: "no-trace-id", // 追踪ID
Duration: int(logItem.Duration), // 请求耗时
TimestampMillis: logItem.CreateTimeSyslog.Format("2006-01-02 15:04:05"), // 创建时间的字符串格式
Timestamp: logItem.CreateTimeSyslog.UnixMilli(), // 创建时间的Unix毫秒时间戳
Tags: map[string]interface{}{
"id": logItem.Id, // 日志主键
"access_uuid": logItem.AccessUuid, // 应用UUID
"access_xzqhdm": logItem.AccessXzqhdm, // 外部传递的区县code
"duration": strconv.FormatInt(logItem.Duration, 10), // 请求耗时
"region_code": logItem.RegionCode, // 区域代码
"service_code": logItem.ServiceCode, // 业务服务代码
"state": logItem.State, // 调用状态
"logClassify": "service", // 业务类型
"service_id": logItem.ServiceId, // 场景服务ID
"url": logItem.RequestUrl, // 节点URL
"theme_type": logItem.ThemeType, // 场景服务类型
"response_status_code": strconv.FormatInt(int64(logItem.ResponseStatusCode), 10), // 接口响应状态码
"remote_ip": logItem.RemoteIp, // 请求IP地址
"request_url": logItem.RequestUrl, // 请求路径
"msg": logItem.LogContent, // 日志内容
"in_params": logItem.InParams, // 请求参数
"out_params": logItem.OutParams, // 响应参数
"in_time": logItem.CreateTimeSyslog.Format("2006-01-02 15:04:05"), // 创建时间
"old_log": "sm_history_log", // 历史日志标识
"scpAppName": "scp-service-manager", // app 标识
"link_no": "0", //
"InitiatorLogKey": "需求方", // 发起方
"ReceiverLogKey": "服务管理", // 接收方
"instance_id": instanceId, // 实例id
},
Q: []string{
fmt.Sprintf("id=%s", logItem.Id), // 日志主键
fmt.Sprintf("access_uuid=%s", logItem.AccessUuid), // 应用UUID
fmt.Sprintf("duration=%s", strconv.FormatInt(logItem.Duration, 10)), // 请求耗时
fmt.Sprintf("region_code=%s", logItem.RegionCode), // 区域代码
fmt.Sprintf("service_code=%s", logItem.ServiceCode), // 业务服务代码
fmt.Sprintf("state=%s", logItem.State), // 调用状态
fmt.Sprintf("logClassify=service"), // 业务类型
fmt.Sprintf("service_id=%s", logItem.ServiceId), // 场景服务ID
fmt.Sprintf("url=%s", logItem.Url), // 节点URL
fmt.Sprintf("theme_type=%s", logItem.ThemeType), // 场景服务类型
fmt.Sprintf("response_status_code=%s", strconv.FormatInt(int64(logItem.ResponseStatusCode), 10)), // 接口响应状态码
fmt.Sprintf("access_xzqhdm=%s", logItem.AccessXzqhdm), // 外部传递的区县code
fmt.Sprintf("remote_ip=%s", logItem.RemoteIp), // 请求IP地址
fmt.Sprintf("request_url=%s", logItem.RequestUrl), // 请求路径
fmt.Sprintf("in_params=%s", logItem.InParams), // 请求参数
fmt.Sprintf("out_params=%s", logItem.OutParams), // 响应参数
fmt.Sprintf("in_time=%s", logItem.CreateTimeSyslog.Format("2006-01-02 15:04:05")), // 创建时间
fmt.Sprintf("old_log=sm_history_log"), // 历史日志标识
fmt.Sprintf("scpAppName=scp-service-manager"), // app 标识
fmt.Sprintf("link_no=0"), //
fmt.Sprintf("InitiatorLogKey=需求方"), // 发起方
fmt.Sprintf("ReceiverLogKey=服务管理"), // 接收方
fmt.Sprintf("instance_id=%s", instanceId), // 实例id
},
}
if len(logItem.LogContent) < 32000 {
esLog.Q = append(esLog.Q, fmt.Sprintf("msg=%s", logItem.LogContent))
}
if serviceInfo, ok := serviceMap[logItem.ServiceId]; ok {
esLog.Tags["service_name"] = serviceInfo.ServiceName
esLog.Tags["theme_name"] = serviceInfo.ThemeName
esLog.Tags["theme_code"] = serviceInfo.ThemeCode
esLog.Q = append(esLog.Q, fmt.Sprintf("service_name=%s", serviceInfo.ServiceName))
esLog.Q = append(esLog.Q, fmt.Sprintf("theme_name=%s", serviceInfo.ThemeName))
esLog.Q = append(esLog.Q, fmt.Sprintf("theme_code=%s", serviceInfo.ThemeCode))
}
if systemName, ok := systemMap[logItem.AccessUuid]; ok {
esLog.Tags["access_system_name"] = systemName
esLog.Q = append(esLog.Q, fmt.Sprintf("access_system_name=%s", systemName))
}
indexStr := config.Elasticsearch.Index + logItem.CreateTimeSyslog.Format("2006-01-02")
fmt.Println(indexStr)
// 写入 Elasticsearch
_, err := esClient.Index().
Index(indexStr).
Id(esLog.ID).
BodyJson(esLog).
Do(context.Background())
if err != nil {
failedIDs = append(failedIDs, logItem.Id)
} else {
successCount++
}
}
// 输出结果
fmt.Printf("Total Logs: %d, Success: %d, Failed: %d\n", totalCount, successCount, len(failedIDs))
if len(failedIDs) > 0 {
fmt.Printf("Failed IDs: %v\n", failedIDs)
}
}
operator_postgre.go 0 → 100644
浏览文件 @ 0dd6348a
package main
import (
"gorm.io/gorm"
)
type Service struct {
Id string `gorm:"primaryKey;type:char(32)" json:"id"`
ServiceName string `gorm:"type:varchar(255);comment:服务名称" json:"service_name"`
Url string `gorm:"type:varchar(255);comment:服务对外地址" json:"url"`
DelFlag int8 `gorm:"comment:是否删除 0：正常 1：已删除" json:"del_flag"`
}
type RelationshipsThemeService struct {
Id string `gorm:"primaryKey;type:char(32)" json:"id"`
ServiceId string `gorm:"type:char(32);comment:服务 id" json:"service_id"`
ThemeId string `gorm:"type:char(32);comment:服务主题 id" json:"theme_id"`
DelFlag int8 `gorm:"comment:是否删除 0：正常 1：已删除" json:"del_flag"`
}
type ServiceTheme struct {
Id string `gorm:"primaryKey;type:char(32)" json:"id"`
ThemeCode string `gorm:"type:varchar(255);comment:服务主题编码" json:"theme_code"`
ThemeName string `gorm:"type:varchar(45);comment:服务主题名称" json:"theme_name"`
DelFlag int8 `gorm:"comment:是否删除 0：正常 1：已删除" json:"del_flag"`
}
type ServiceResult struct {
Id string `json:"id"`
ServiceName string `json:"service_name"`
Url string `json:"url"`
ThemeCode string `json:"theme_code"`
ThemeName string `json:"theme_name"`
}
func GetServiceThemeInfo(schemaName string, db *gorm.DB) (map[string]ServiceResult, error) {
var results []ServiceResult
serviceMap := make(map[string]ServiceResult)
// 动态生成表名
serviceInfoTable := schemaName + ".service_info"
relationshipsTable := schemaName + ".relationships_theme_service"
themeTable := schemaName + ".theme_service"
// 查询所有符合条件的 service_info, relationships_theme_service, 和 service_theme 表
err := db.Table(serviceInfoTable).
Select("service_info.id, service_info.service_name, service_info.url, theme_service.theme_code, theme_service.theme_name").
Joins("JOIN " + relationshipsTable + " ON service_info.id = " + relationshipsTable + ".service_id").
Joins("JOIN " + themeTable + " ON " + relationshipsTable + ".theme_id = " + themeTable + ".id").
Where("service_info.del_flag = 0 AND " + relationshipsTable + ".del_flag = 0 AND " + themeTable + ".del_flag = 0").
Scan(&results).Error
if err != nil {
return nil, err
}
// 将结果按 service_id 存入 map
for _, result := range results {
serviceMap[result.Id] = result
}
return serviceMap, nil
}
// ServiceAccess 服务接入信息
type ServiceAccess struct {
Id string `gorm:"primaryKey;type:char(32)" json:"id"` // 主键id
BuilderId string `gorm:"type:char(32);comment:建设单位id 需求方id" json:"builder_id"` // 建设单位id 需求方id
AccessUuid string `gorm:"type:varchar(70);comment:接入唯一码" json:"access_uuid"` // 接入唯一码
DelFlag int8 `gorm:"comment:是否删除 0：正常 1：已删除" json:"del_flag"` // 是否删除 0：正常 1：已删除
}
// AccessBuilder 建设单位信息
type AccessBuilder struct {
Id string `gorm:"primaryKey;type:char(32)" json:"id"` // 主键id
SystemName string `gorm:"type:varchar(45);comment:系统名称" json:"system_name"` // 系统名称
DelFlag int8 `gorm:"comment:是否删除 0：正常 1：已删除" json:"del_flag"` // 是否删除 0：正常 1：已删除
}
// GetServiceAccessInfo 查询服务接入信息和建设单位信息
func GetServiceAccessInfo(db *gorm.DB) (map[string]string, error) {
var results []struct {
AccessUuid string
SystemName string
}
// 执行查询，关联 service_access 和 access_builder 表，并且保证各个表的 del_flag = 0
err := db.Table("service_access").
Select("service_access.access_uuid, access_builder.system_name").
Joins("JOIN access_builder ON service_access.builder_id = access_builder.id").
Where("service_access.del_flag = 0 AND access_builder.del_flag = 0").
Scan(&results).Error
if err != nil {
return nil, err
}
// 将查询结果存入 map[string]string
resultMap := make(map[string]string)
for _, result := range results {
resultMap[result.AccessUuid] = result.SystemName
}
return resultMap, nil
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论