Directory tree
repo ├── go-apk │ └── README.md ├── go-cache │ ├── cache.go │ ├── client.go │ └── config.go ├── go-captcha │ ├── test │ │ └── test.go │ ├── client.go │ ├── gocaptcha.go │ └── readme.md ├── go-const │ └── http.go ├── go-context │ ├── context.go │ ├── context_test.go │ ├── context_unix.go │ └── readme.md ├── go-db │ ├── go-clickhouse │ │ ├── test │ │ │ └── main.go │ │ ├── clickhouse.go │ │ ├── client.go │ │ ├── config.go │ │ └── readme.md │ ├── go-es │ │ ├── eslog │ │ │ └── log.go │ │ ├── client.go │ │ ├── client_doc_create.go │ │ ├── client_doc_create_test.go │ │ ├── client_doc_delete.go │ │ ├── client_doc_select_test.go │ │ ├── client_doc_selete.go │ │ ├── client_doc_update.go │ │ ├── client_doc_update_test.go │ │ ├── client_index.go │ │ ├── client_index_test.go │ │ ├── config.go │ │ ├── es.go │ │ └── readme.md │ ├── go-esnew │ │ ├── test │ │ │ └── test.go │ │ ├── client.go │ │ ├── client_doc_select.go │ │ ├── readme.md │ │ └── types.go │ ├── go-mongo │ │ ├── test │ │ │ └── main.go │ │ ├── client.go │ │ ├── config.go │ │ ├── mongo.go │ │ └── readme.md │ ├── go-redis │ │ ├── go-redisc │ │ │ ├── client.go │ │ │ ├── config.go │ │ │ ├── redis.go │ │ │ └── utils.go │ │ ├── test │ │ │ └── main.go │ │ ├── client.go │ │ ├── config.go │ │ ├── README.md │ │ └── redis.go │ ├── gogorm │ │ ├── client.go │ │ ├── clientex.go │ │ ├── config.go │ │ ├── go_gorm.go │ │ └── ReadMe.md │ ├── test │ │ └── main.go │ ├── pageex.go │ ├── readme.md │ └── select.go ├── go-doc │ └── Event 结构体字段说明文档.md ├── go-error │ ├── test │ │ └── main.go │ ├── errCode.go │ ├── errMsg.go │ ├── errors.go │ ├── httpCode.go │ ├── httpErrCode.go │ ├── readme.md │ └── util.go ├── go-etcd │ ├── client.go │ ├── config.go │ ├── etcd.go │ ├── etcd_test.go │ └── readme.md ├── go-event │ ├── test │ │ └── main.go │ ├── default.go │ ├── event.go │ ├── message.go │ └── README.md ├── go-file │ ├── test │ │ └── main.go │ ├── bigfile.go │ ├── binary.go │ ├── download.go │ ├── file.go │ ├── file_exist.go │ ├── file_lock.go │ ├── file_md5.go │ ├── file_readline.go │ ├── filelock_unix.go │ ├── filelock_windows.go │ ├── readme.md │ ├── types.go │ ├── upload.go │ └── zip.go ├── go-grpc │ └── readme.md ├── go-http │ ├── gohttpx │ │ ├── ca_cert.go │ │ ├── const.go │ │ ├── http.go │ │ ├── option.go │ │ ├── readme.md │ │ ├── request.go │ │ └── tls.go │ ├── gowebsocket │ │ ├── client.go │ │ ├── readme.md │ │ ├── server.go │ │ ├── types.go │ │ └── websocket.go │ ├── test │ │ └── main.go │ ├── http.go │ ├── readme.md │ ├── types.go │ └── utils.go ├── go-ip │ ├── test │ │ └── main.go │ ├── cidr.go │ ├── client.go │ ├── config.go │ ├── goip.go │ ├── README.md │ └── types.go ├── go-job │ ├── test │ │ └── main.go │ ├── gojob.go │ └── readme.md ├── go-lock │ ├── test │ │ └── main.go │ ├── golock.go │ └── readme.md ├── go-log │ ├── adapters │ │ ├── es.go │ │ ├── file.go │ │ ├── file_options.go │ │ └── kafka.go │ ├── test │ │ ├── console_test.go │ │ ├── file_test.go │ │ └── log_test.go │ ├── adapter.go │ ├── console.go │ ├── entry.go │ ├── level.go │ ├── log.go │ ├── logger.go │ ├── message.go │ └── readme.md ├── go-mail │ ├── config.go │ ├── default.go │ ├── mail.go │ ├── mail_test.go │ ├── message.go │ └── readme.md ├── go-marketing │ ├── go-admob │ │ ├── client.go │ │ ├── config.go │ │ ├── goadmob.go │ │ ├── goadmob_test.go │ │ └── ReadMe.md │ ├── go-amazon │ │ └── ip-range │ │ └── ip_range.go │ ├── go-baidu │ │ └── ReadMe.md │ ├── go-googleads │ │ └── ReadMe.md │ ├── go-meta │ │ ├── test │ │ │ ├── gometa_test.go │ │ │ └── test.json │ │ ├── audience.go │ │ ├── audience_types.go │ │ ├── client.go │ │ ├── gometa.go │ │ ├── market.go │ │ ├── market_types.go │ │ └── ReadMe.md │ ├── go-toutiao │ │ └── ReadMe.md │ ├── goattribution │ │ ├── cryptography │ │ │ ├── aes_crt.go │ │ │ ├── aes_crt_test.go │ │ │ ├── aes_gcm.go │ │ │ ├── aes_gcm_test.go │ │ │ ├── hmacsha.go │ │ │ └── sorted_params.go │ │ ├── appsflyer.go │ │ ├── attribute.go │ │ ├── bigo.go │ │ ├── common.go │ │ ├── google.go │ │ ├── manager.go │ │ ├── manager_test.go │ │ ├── meta.go │ │ ├── organic.go │ │ ├── readme.md │ │ ├── referer.go │ │ ├── utils.go │ │ └── yandex.go │ └── ReadMe.md ├── go-message │ ├── goding │ │ └── ding.go │ ├── gofeishu │ │ └── feishu.go │ ├── gotg │ │ └── telegram.go │ ├── dingding_test.go │ ├── feishu_test.go │ ├── go_message.go │ ├── README.md │ └── telegram_test.go ├── go-mq │ ├── go-asynq │ │ ├── go-asynqc │ │ │ ├── client.go │ │ │ ├── goasynq_c.go │ │ │ ├── inspector.go │ │ │ └── server.go │ │ ├── client.go │ │ ├── goasynq.go │ │ ├── readme.md │ │ └── server.go │ ├── go-kafka │ │ ├── adapter.go │ │ ├── adapter_consumer.go │ │ ├── adapter_consumer_group.go │ │ ├── adapter_producer.go │ │ ├── client.go │ │ ├── config.go │ │ ├── kafka.go │ │ ├── option.go │ │ ├── readme.md │ │ └── type.go │ ├── go-mqtt │ │ ├── test │ │ │ └── main.go │ │ ├── client.go │ │ ├── gomqtt.go │ │ └── readme.md │ ├── go-rabbitmq │ │ └── readme.md │ ├── go-rocketmq │ │ ├── test │ │ │ └── admin │ │ │ └── main.go │ │ └── readme.md │ ├── gomq.go │ └── readme.md ├── go-oss │ ├── go-alioss │ │ ├── test │ │ │ └── main.go │ │ ├── ali-oss.go │ │ ├── config.go │ │ ├── readme.md │ │ └── uploader.go │ ├── go-minio │ │ ├── test │ │ │ └── main.go │ │ ├── config.go │ │ ├── gominio.go │ │ ├── README.md │ │ └── uploader.go │ └── readme.md ├── go-pay │ ├── go-alipay │ │ └── README.md │ └── go-wechat │ ├── test │ │ └── main.go │ ├── README.md │ └── utils.go ├── go-pool │ ├── test │ │ └── main.go │ ├── gopool.go │ └── readme.md ├── go-shell │ └── tag.sh ├── go-sso │ ├── go-jwt │ │ ├── test │ │ │ └── main.go │ │ ├── test1 │ │ │ └── test.go │ │ ├── client.go │ │ └── gojwt.go │ └── go-oauth │ ├── client.go │ ├── go-oauth.go │ ├── go-oauth_test.go │ ├── readme.md │ ├── types.go │ └── utils.go ├── go-test │ └── readme.md ├── go-tgbot │ ├── test │ │ └── main.go │ ├── tgext │ │ └── tgext.go │ ├── readme.md │ ├── tgbot.go │ ├── tgbots.go │ └── types.go ├── go-utils │ ├── convert_type │ │ ├── converter.go │ │ ├── utils.go │ │ └── value_type.go │ ├── gobean │ │ ├── bean.go │ │ ├── constants.go │ │ └── readme.md │ ├── gocrypto │ │ ├── crypto.go │ │ └── utils.go │ ├── gojson │ │ ├── gojson.go │ │ └── readme.md │ ├── goprometheus │ │ ├── goprometheusx │ │ │ ├── agent.go │ │ │ ├── config.go │ │ │ ├── counter.go │ │ │ ├── gauge.go │ │ │ ├── histogram.go │ │ │ ├── metric.go │ │ │ └── prometheus.go │ │ ├── client.go │ │ ├── config.go │ │ ├── consts.go │ │ ├── prometheus.go │ │ ├── query.go │ │ ├── readme.md │ │ ├── svr_member_level_user_count.go │ │ ├── sys_bandwidth_in.go │ │ ├── sys_bandwidth_out.go │ │ ├── sys_cpu_core_count.go │ │ ├── sys_cpu_usage_rate.go │ │ ├── sys_disk_total.go │ │ ├── sys_disk_usage.go │ │ ├── sys_memory_total.go │ │ ├── sys_memory_usage.go │ │ ├── sys_total_bandwidth_in.go │ │ ├── sys_total_bandwidth_out.go │ │ ├── sys_total_traffic_in.go │ │ ├── sys_total_traffic_out.go │ │ ├── sys_traffic_in.go │ │ ├── sys_traffic_out.go │ │ ├── sys_up.go │ │ ├── types.go │ │ └── utils.go │ ├── gorandom │ │ ├── gorandom.go │ │ ├── gorandomnorepeat.go │ │ └── utils.go │ ├── gotime │ │ ├── readme.md │ │ ├── time.go │ │ ├── timer.go │ │ └── timex.go │ ├── gozip │ │ ├── gozip.go │ │ ├── types.go │ │ └── utils.go │ ├── array.go │ ├── async.go │ ├── bigint.go │ ├── captcha.go │ ├── crypto.go │ ├── email.go │ ├── encoding.go │ ├── id_gen.go │ ├── id_snow_flake.go │ ├── id_snow_flake_test.go │ ├── idcode.go │ ├── idcode_test.go │ ├── map.go │ ├── noncestr.go │ ├── params.go │ ├── pinyin.go │ ├── pinyin_resource.go │ ├── readme.md │ ├── safeslice.go │ ├── string.go │ ├── time.go │ ├── timex.go │ ├── utils.go │ ├── validation.go │ └── xml.go ├── go-xlsx │ ├── csv_reader.go │ ├── csv_writer.go │ ├── readme.md │ ├── types.go │ ├── xlsx_reader.go │ ├── xlsx_test.go │ └── xlsx_writer.go ├── go-zero │ ├── etcd-configurator │ │ ├── test │ │ │ └── config_test.go │ │ └── config_center.go │ ├── transactionex │ │ ├── model.go │ │ ├── ReadMe.md │ │ ├── transaction.go │ │ └── types.go │ ├── grpc-limit.go │ └── loggerInterceptor.go ├── goio │ ├── go-test │ │ └── main.go │ ├── goi │ │ ├── clickhouse.go │ │ ├── db.go │ │ ├── es.go │ │ ├── etcd.go │ │ ├── goip.go │ │ ├── gominio.go │ │ ├── jwt.go │ │ ├── kafka.go │ │ ├── mongo.go │ │ ├── readme.md │ │ └── redis.go │ ├── server │ │ ├── config.go │ │ ├── server.go │ │ ├── server_encryption.go │ │ ├── server_encryption_test.go │ │ ├── server_grpc.go │ │ ├── server_handler.go │ │ ├── server_options.go │ │ ├── server_response.go │ │ ├── server_token.go │ │ ├── server_upload.go │ │ ├── server_utils.go │ │ └── server_validator.go │ ├── server-case │ │ ├── config │ │ │ └── config.go │ │ ├── controller │ │ │ └── common-controller │ │ │ ├── captcha.go │ │ │ ├── file_download.go │ │ │ ├── health.go │ │ │ └── user_login.go │ │ ├── etc │ │ │ ├── api-local.yaml │ │ │ ├── api-prod.yaml │ │ │ └── api-test.yaml │ │ ├── router │ │ │ ├── middleware.go │ │ │ └── router.go │ │ └── main.go │ ├── test │ │ └── main.go │ ├── env.go │ ├── error_state.go │ ├── flag.go │ ├── goio.go │ ├── README.md │ └── status.go ├── .deepsource.toml ├── .gitignore ├── .pid ├── go.mod ├── go.sum ├── LICENSE ├── main.go └── README.md
Table of contents (392)
- .deepsource.toml (101 B)
- .gitignore (24 B)
- .pid (5 B)
- LICENSE (11.1 KiB)
- README.md (8.5 KiB)
- go-apk/README.md (152 B)
- go-cache/cache.go (722 B)
- go-cache/client.go (883 B)
- go-cache/config.go (287 B)
- go-captcha/client.go (1.1 KiB)
- go-captcha/gocaptcha.go (5.3 KiB)
- go-captcha/readme.md (190 B)
- go-captcha/test/test.go (4.8 KiB)
- go-const/http.go (16 B)
- go-context/context.go (2.0 KiB)
- go-context/context_test.go (388 B)
- go-context/context_unix.go (723 B)
- go-context/readme.md (299 B)
- go-db/go-clickhouse/clickhouse.go (1.4 KiB)
- go-db/go-clickhouse/client.go (8.1 KiB)
- go-db/go-clickhouse/config.go (1.6 KiB)
- go-db/go-clickhouse/readme.md (4.3 KiB)
- go-db/go-clickhouse/test/main.go (791 B)
- go-db/go-es/client.go (1.5 KiB)
- go-db/go-es/client_doc_create.go (842 B)
- go-db/go-es/client_doc_create_test.go (920 B)
- go-db/go-es/client_doc_delete.go (1.0 KiB)
- go-db/go-es/client_doc_select_test.go (2.8 KiB)
- go-db/go-es/client_doc_selete.go (4.2 KiB)
- go-db/go-es/client_doc_update.go (3.7 KiB)
- go-db/go-es/client_doc_update_test.go (1.9 KiB)
- go-db/go-es/client_index.go (2.2 KiB)
- go-db/go-es/client_index_test.go (946 B)
- go-db/go-es/config.go (325 B)
- go-db/go-es/es.go (984 B)
- go-db/go-es/eslog/log.go (492 B)
- go-db/go-es/readme.md (123 B)
- go-db/go-esnew/client.go (3.2 KiB)
- go-db/go-esnew/client_doc_select.go (5.2 KiB)
- go-db/go-esnew/readme.md (79 B)
- go-db/go-esnew/test/test.go (2.5 KiB)
- go-db/go-esnew/types.go (146 B)
- go-db/go-mongo/client.go (1.3 KiB)
- go-db/go-mongo/config.go (490 B)
- go-db/go-mongo/mongo.go (1.1 KiB)
- go-db/go-mongo/readme.md (16 B)
- go-db/go-mongo/test/main.go (966 B)
- go-db/go-redis/README.md (702 B)
- go-db/go-redis/client.go (18.7 KiB)
- go-db/go-redis/config.go (1.9 KiB)
- go-db/go-redis/go-redisc/client.go (19.2 KiB)
- go-db/go-redis/go-redisc/config.go (1.9 KiB)
- go-db/go-redis/go-redisc/redis.go (1.2 KiB)
- go-db/go-redis/go-redisc/utils.go (7.7 KiB)
- go-db/go-redis/redis.go (1.2 KiB)
- go-db/go-redis/test/main.go (605 B)
- go-db/gogorm/ReadMe.md (3.0 KiB)
- go-db/gogorm/client.go (1.5 KiB)
- go-db/gogorm/clientex.go (1.7 KiB)
- go-db/gogorm/config.go (837 B)
- go-db/gogorm/go_gorm.go (3.2 KiB)
- go-db/pageex.go (2.4 KiB)
- go-db/readme.md (29 B)
- go-db/select.go (3.3 KiB)
- go-db/test/main.go (6.6 KiB)
- go-doc/Event 结构体字段说明文档.md (7.4 KiB)
- go-error/errCode.go (921 B)
- go-error/errMsg.go (1.7 KiB)
- go-error/errors.go (3.9 KiB)
- go-error/httpCode.go (1.9 KiB)
- go-error/httpErrCode.go (502 B)
- go-error/readme.md (625 B)
- go-error/test/main.go (580 B)
- go-error/util.go (1.5 KiB)
- go-etcd/client.go (7.7 KiB)
- go-etcd/config.go (541 B)
- go-etcd/etcd.go (2.4 KiB)
- go-etcd/etcd_test.go (1.5 KiB)
- go-etcd/readme.md (118 B)
- go-event/README.md (507 B)
- go-event/default.go (370 B)
- go-event/event.go (2.7 KiB)
- go-event/message.go (143 B)
- go-event/test/main.go (999 B)
- go-file/bigfile.go (2.7 KiB)
- go-file/binary.go (1.2 KiB)
- go-file/download.go (2.4 KiB)
- go-file/file.go (3.4 KiB)
- go-file/file_exist.go (168 B)
- go-file/file_lock.go (522 B)
- go-file/file_md5.go (2.0 KiB)
- go-file/file_readline.go (596 B)
- go-file/filelock_unix.go (240 B)
- go-file/filelock_windows.go (152 B)
- go-file/readme.md (938 B)
- go-file/test/main.go (3.4 KiB)
- go-file/types.go (1014 B)
- go-file/upload.go (5.3 KiB)
- go-file/zip.go (1.2 KiB)
- go-grpc/readme.md (37 B)
- go-http/gohttpx/const.go (180 B)
- go-http/gohttpx/http.go (3.6 KiB)
- go-http/gohttpx/option.go (828 B)
- go-http/gohttpx/readme.md (427 B)
- go-http/gohttpx/request.go (3.1 KiB)
- go-http/gohttpx/tls.go (534 B)
- go-http/gowebsocket/client.go (6.8 KiB)
- go-http/gowebsocket/readme.md (1.4 KiB)
- go-http/gowebsocket/types.go (488 B)
- go-http/gowebsocket/websocket.go (2.5 KiB)
- go-http/http.go (5.7 KiB)
- go-http/readme.md (1.0 KiB)
- go-http/test/main.go (1.6 KiB)
- go-http/types.go (3.9 KiB)
- go-http/utils.go (2.5 KiB)
- go-ip/README.md (1.2 KiB)
- go-ip/cidr.go (2.1 KiB)
- go-ip/client.go (991 B)
- go-ip/config.go (299 B)
- go-ip/goip.go (4.7 KiB)
- go-ip/test/main.go (1.2 KiB)
- go-ip/types.go (1.0 KiB)
- go-job/gojob.go (7.7 KiB)
- go-job/readme.md (2.4 KiB)
- go-job/test/main.go (2.5 KiB)
- go-lock/golock.go (1.5 KiB)
- go-lock/readme.md (970 B)
- go-lock/test/main.go (619 B)
- go-log/adapter.go (63 B)
- go-log/adapters/es.go (862 B)
- go-log/adapters/file.go (4.5 KiB)
- go-log/adapters/file_options.go (1.2 KiB)
- go-log/adapters/kafka.go (958 B)
- go-log/console.go (1.2 KiB)
- go-log/entry.go (3.3 KiB)
- go-log/level.go (676 B)
- go-log/log.go (1.3 KiB)
- go-log/logger.go (1.5 KiB)
- go-log/message.go (970 B)
- go-log/readme.md (1.8 KiB)
- go-log/test/console_test.go (530 B)
- go-log/test/file_test.go (830 B)
- go-log/test/log_test.go (569 B)
- go-mail/config.go (318 B)
- go-mail/default.go (145 B)
- go-mail/mail.go (1.6 KiB)
- go-mail/mail_test.go (1.3 KiB)
- go-mail/message.go (761 B)
- go-mail/readme.md (30 B)
- go-marketing/ReadMe.md (478 B)
- go-marketing/go-admob/ReadMe.md (6.7 KiB)
- go-marketing/go-admob/client.go (1.1 KiB)
- go-marketing/go-admob/config.go (608 B)
- go-marketing/go-admob/goadmob.go (12.4 KiB)
- go-marketing/go-admob/goadmob_test.go (4.1 KiB)
- go-marketing/go-amazon/ip-range/ip_range.go (1.6 KiB)
- go-marketing/go-baidu/ReadMe.md (22 B)
- go-marketing/go-googleads/ReadMe.md (27 B)
- go-marketing/go-meta/ReadMe.md (894 B)
- go-marketing/go-meta/audience.go (4.2 KiB)
- go-marketing/go-meta/audience_types.go (6.3 KiB)
- go-marketing/go-meta/client.go (1.1 KiB)
- go-marketing/go-meta/gometa.go (3.0 KiB)
- go-marketing/go-meta/market.go (4.2 KiB)
- go-marketing/go-meta/market_types.go (6.7 KiB)
- go-marketing/go-meta/test/gometa_test.go (1.0 KiB)
- go-marketing/go-meta/test/test.json (3.4 KiB)
- go-marketing/go-toutiao/ReadMe.md (24 B)
- go-marketing/goattribution/appsflyer.go (1.5 KiB)
- go-marketing/goattribution/attribute.go (2.7 KiB)
- go-marketing/goattribution/bigo.go (535 B)
- go-marketing/goattribution/common.go (657 B)
- go-marketing/goattribution/cryptography/aes_crt.go (431 B)
- go-marketing/goattribution/cryptography/aes_crt_test.go (2.9 KiB)
- go-marketing/goattribution/cryptography/aes_gcm.go (513 B)
- go-marketing/goattribution/cryptography/aes_gcm_test.go (3.1 KiB)
- go-marketing/goattribution/cryptography/hmacsha.go (358 B)
- go-marketing/goattribution/cryptography/sorted_params.go (1001 B)
- go-marketing/goattribution/google.go (579 B)
- go-marketing/goattribution/manager.go (1.5 KiB)
- go-marketing/goattribution/manager_test.go (5.9 KiB)
- go-marketing/goattribution/meta.go (3.4 KiB)
- go-marketing/goattribution/organic.go (669 B)
- go-marketing/goattribution/readme.md (81 B)
- go-marketing/goattribution/referer.go (603 B)
- go-marketing/goattribution/utils.go (498 B)
- go-marketing/goattribution/yandex.go (594 B)
- go-message/README.md (638 B)
- go-message/dingding_test.go (664 B)
- go-message/feishu_test.go (622 B)
- go-message/go_message.go (1.6 KiB)
- go-message/goding/ding.go (3.6 KiB)
- go-message/gofeishu/feishu.go (818 B)
- go-message/gotg/telegram.go (325 B)
- go-message/telegram_test.go (573 B)
- go-mq/go-asynq/client.go (4.5 KiB)
- go-mq/go-asynq/go-asynqc/client.go (2.6 KiB)
- go-mq/go-asynq/go-asynqc/goasynq_c.go (2.8 KiB)
- go-mq/go-asynq/go-asynqc/inspector.go (3.5 KiB)
- go-mq/go-asynq/go-asynqc/server.go (4.4 KiB)
- go-mq/go-asynq/goasynq.go (1.8 KiB)
- go-mq/go-asynq/readme.md (271 B)
- go-mq/go-asynq/server.go (3.2 KiB)
- go-mq/go-kafka/adapter.go (1007 B)
- go-mq/go-kafka/adapter_consumer.go (3.2 KiB)
- go-mq/go-kafka/adapter_consumer_group.go (3.3 KiB)
- go-mq/go-kafka/adapter_producer.go (5.7 KiB)
- go-mq/go-kafka/client.go (7.3 KiB)
- go-mq/go-kafka/config.go (1.1 KiB)
- go-mq/go-kafka/kafka.go (1.6 KiB)
- go-mq/go-kafka/option.go (387 B)
- go-mq/go-kafka/readme.md (1.4 KiB)
- go-mq/go-kafka/type.go (1.4 KiB)
- go-mq/go-mqtt/client.go (4.1 KiB)
- go-mq/go-mqtt/gomqtt.go (1.0 KiB)
- go-mq/go-mqtt/readme.md (389 B)
- go-mq/go-mqtt/test/main.go (2.7 KiB)
- go-mq/go-rabbitmq/readme.md (12 B)
- go-mq/go-rocketmq/readme.md (50 B)
- go-mq/go-rocketmq/test/admin/main.go (2.7 KiB)
- go-mq/gomq.go (2.3 KiB)
- go-mq/readme.md (29 B)
- go-oss/go-alioss/ali-oss.go (1.6 KiB)
- go-oss/go-alioss/config.go (513 B)
- go-oss/go-alioss/readme.md (77 B)
- go-oss/go-alioss/test/main.go (1.6 KiB)
- go-oss/go-alioss/uploader.go (1.5 KiB)
- go-oss/go-minio/README.md (3.0 KiB)
- go-oss/go-minio/config.go (611 B)
- go-oss/go-minio/gominio.go (1.2 KiB)
- go-oss/go-minio/test/main.go (3.0 KiB)
- go-oss/go-minio/uploader.go (3.2 KiB)
- go-oss/readme.md (6 B)
- go-pay/go-alipay/README.md (90 B)
- go-pay/go-wechat/README.md (699 B)
- go-pay/go-wechat/test/main.go (2.9 KiB)
- go-pay/go-wechat/utils.go (640 B)
- go-pool/gopool.go (2.7 KiB)
- go-pool/readme.md (267 B)
- go-pool/test/main.go (924 B)
- go-shell/tag.sh (1.4 KiB)
- go-sso/go-jwt/client.go (991 B)
- go-sso/go-jwt/gojwt.go (3.1 KiB)
- go-sso/go-jwt/test/main.go (764 B)
- go-sso/go-jwt/test1/test.go (357 B)
- go-sso/go-oauth/client.go (1.1 KiB)
- go-sso/go-oauth/go-oauth.go (3.7 KiB)
- go-sso/go-oauth/go-oauth_test.go (834 B)
- go-sso/go-oauth/readme.md (81 B)
- go-sso/go-oauth/types.go (961 B)
- go-sso/go-oauth/utils.go (1.8 KiB)
- go-test/readme.md (156 B)
- go-tgbot/readme.md (2.8 KiB)
- go-tgbot/test/main.go (2.3 KiB)
- go-tgbot/tgbot.go (5.9 KiB)
- go-tgbot/tgbots.go (1015 B)
- go-tgbot/tgext/tgext.go (3.1 KiB)
- go-tgbot/types.go (1.4 KiB)
- go-utils/array.go (2.6 KiB)
- go-utils/async.go (2.4 KiB)
- go-utils/bigint.go (1.8 KiB)
- go-utils/captcha.go (2.0 KiB)
- go-utils/convert_type/converter.go (20.0 KiB)
- go-utils/convert_type/utils.go (502 B)
- go-utils/convert_type/value_type.go (3.3 KiB)
- go-utils/crypto.go (8.5 KiB)
- go-utils/email.go (702 B)
- go-utils/encoding.go (732 B)
- go-utils/gobean/bean.go (3.1 KiB)
- go-utils/gobean/constants.go (694 B)
- go-utils/gobean/readme.md (26 B)
- go-utils/gocrypto/crypto.go (12.3 KiB)
- go-utils/gocrypto/utils.go (4.1 KiB)
- go-utils/gojson/gojson.go (1.4 KiB)
- go-utils/gojson/readme.md (221 B)
- go-utils/goprometheus/client.go (1.1 KiB)
- go-utils/goprometheus/config.go (397 B)
- go-utils/goprometheus/consts.go (5.1 KiB)
- go-utils/goprometheus/goprometheusx/agent.go (1.0 KiB)
- go-utils/goprometheus/goprometheusx/config.go (202 B)
- go-utils/goprometheus/goprometheusx/counter.go (1.1 KiB)
- go-utils/goprometheus/goprometheusx/gauge.go (1.2 KiB)
- go-utils/goprometheus/goprometheusx/histogram.go (1.1 KiB)
- go-utils/goprometheus/goprometheusx/metric.go (186 B)
- go-utils/goprometheus/goprometheusx/prometheus.go (1.2 KiB)
- go-utils/goprometheus/prometheus.go (891 B)
- go-utils/goprometheus/query.go (1.4 KiB)
- go-utils/goprometheus/readme.md (1.0 KiB)
- go-utils/goprometheus/svr_member_level_user_count.go (1.9 KiB)
- go-utils/goprometheus/sys_bandwidth_in.go (1.6 KiB)
- go-utils/goprometheus/sys_bandwidth_out.go (1.6 KiB)
- go-utils/goprometheus/sys_cpu_core_count.go (1.5 KiB)
- go-utils/goprometheus/sys_cpu_usage_rate.go (1.5 KiB)
- go-utils/goprometheus/sys_disk_total.go (1.3 KiB)
- go-utils/goprometheus/sys_disk_usage.go (1.6 KiB)
- go-utils/goprometheus/sys_memory_total.go (1.3 KiB)
- go-utils/goprometheus/sys_memory_usage.go (1.5 KiB)
- go-utils/goprometheus/sys_total_bandwidth_in.go (1.6 KiB)
- go-utils/goprometheus/sys_total_bandwidth_out.go (1.6 KiB)
- go-utils/goprometheus/sys_total_traffic_in.go (1.5 KiB)
- go-utils/goprometheus/sys_total_traffic_out.go (1.6 KiB)
- go-utils/goprometheus/sys_traffic_in.go (1.6 KiB)
- go-utils/goprometheus/sys_traffic_out.go (1.6 KiB)
- go-utils/goprometheus/sys_up.go (1.1 KiB)
- go-utils/goprometheus/types.go (2.0 KiB)
- go-utils/goprometheus/utils.go (833 B)
- go-utils/gorandom/gorandom.go (5.7 KiB)
- go-utils/gorandom/gorandomnorepeat.go (2.3 KiB)
- go-utils/gorandom/utils.go (4.7 KiB)
- go-utils/gotime/readme.md (33 B)
- go-utils/gotime/time.go (3.1 KiB)
- go-utils/gotime/timer.go (2.3 KiB)
- go-utils/gotime/timex.go (8.7 KiB)
- go-utils/gozip/gozip.go (4.1 KiB)
- go-utils/gozip/types.go (14 B)
- go-utils/gozip/utils.go (1.2 KiB)
- go-utils/id_gen.go (480 B)
- go-utils/id_snow_flake.go (2.0 KiB)
- go-utils/id_snow_flake_test.go (967 B)
- go-utils/idcode.go (2.2 KiB)
- go-utils/idcode_test.go (583 B)
- go-utils/map.go (269 B)
- go-utils/noncestr.go (290 B)
- go-utils/params.go (3.2 KiB)
- go-utils/pinyin.go (3.4 KiB)
- go-utils/readme.md (2.7 KiB)
- go-utils/safeslice.go (1.3 KiB)
- go-utils/string.go (1.5 KiB)
- go-utils/time.go (3.0 KiB)
- go-utils/timex.go (8.9 KiB)
- go-utils/utils.go (7.3 KiB)
- go-utils/validation.go (1.0 KiB)
- go-utils/xml.go (740 B)
- go-xlsx/csv_reader.go (4.9 KiB)
- go-xlsx/csv_writer.go (5.6 KiB)
- go-xlsx/readme.md (109 B)
- go-xlsx/types.go (616 B)
- go-xlsx/xlsx_reader.go (909 B)
- go-xlsx/xlsx_test.go (1.1 KiB)
- go-xlsx/xlsx_writer.go (2.6 KiB)
- go-zero/etcd-configurator/config_center.go (1.1 KiB)
- go-zero/etcd-configurator/test/config_test.go (980 B)
- go-zero/grpc-limit.go (905 B)
- go-zero/loggerInterceptor.go (825 B)
- go-zero/transactionex/ReadMe.md (665 B)
- go-zero/transactionex/model.go (731 B)
- go-zero/transactionex/transaction.go (916 B)
- go-zero/transactionex/types.go (110 B)
- go.mod (10.1 KiB)
- goio/README.md (668 B)
- goio/env.go (1.7 KiB)
- goio/error_state.go (374 B)
- goio/flag.go (616 B)
- goio/go-test/main.go (543 B)
- goio/goi/clickhouse.go (514 B)
- goio/goi/db.go (290 B)
- goio/goi/es.go (282 B)
- goio/goi/etcd.go (155 B)
- goio/goi/goip.go (139 B)
- goio/goi/gominio.go (165 B)
- goio/goi/jwt.go (152 B)
- goio/goi/kafka.go (363 B)
- goio/goi/mongo.go (342 B)
- goio/goi/readme.md (18 B)
- goio/goi/redis.go (333 B)
- goio/goio.go (474 B)
- goio/server/config.go (1.9 KiB)
- goio/server/server.go (4.3 KiB)
- goio/server/server_encryption.go (736 B)
- goio/server/server_encryption_test.go (657 B)
- goio/server/server_grpc.go (1.4 KiB)
- goio/server/server_handler.go (1.4 KiB)
- goio/server/server_options.go (2.4 KiB)
- goio/server/server_response.go (4.0 KiB)
- goio/server/server_token.go (1.4 KiB)
- goio/server/server_upload.go (1.2 KiB)
- goio/server/server_utils.go (2.4 KiB)
- goio/server/server_validator.go (687 B)
- goio/server-case/config/config.go (1.9 KiB)
- goio/server-case/controller/common-controller/captcha.go (532 B)
- goio/server-case/controller/common-controller/file_download.go (1.0 KiB)
- goio/server-case/controller/common-controller/health.go (235 B)
- goio/server-case/controller/common-controller/user_login.go (1.6 KiB)
- goio/server-case/etc/api-local.yaml (500 B)
- goio/server-case/etc/api-prod.yaml (500 B)
- goio/server-case/etc/api-test.yaml (500 B)
- goio/server-case/main.go (2.3 KiB)
- goio/server-case/router/middleware.go (639 B)
- goio/server-case/router/router.go (534 B)
- goio/status.go (79 B)
- goio/test/main.go (524 B)
- main.go (751 B)
Skipped items
Skipped binaries (1)
go-http/gowebsocket/server.go(15.1 KiB)
Skipped large files (3)
go-http/gohttpx/ca_cert.go(340.7 KiB)go-utils/pinyin_resource.go(726.4 KiB)go.sum(140.7 KiB)
.deepsource.toml (101 B)
version = 1
[[analyzers]]
name = "go"
[analyzers.meta]
import_root = "github.com/gif-gif/go.io"
.gitignore (24 B)
.idea
logs
.pid
go-test
.pid (5 B)
21172
LICENSE (11.1 KiB)
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
README.md (8.5 KiB)
go.io
Golang Development Framework Continuously Developing and Updating
平台
- 支持 Linux、MacOS、Windows 等主流操作系统;
- 支持 32 位和 64 位 CPU 架构;
设计目标
- goio 提供了常用库封装,支持必要的简洁使用功能,在其之上可以进二次开发,以提供更好的代码维护;
- 以跨平台跨项目为首要原则,以减少二次开发的成本;
- 各个模块逻辑保持唯一不重复,但模块之间相互便捷使用实现复杂逻辑开发
- 所有模块包名以go开头
开发规范
- dev 分之开发,跑测试case,确定没问题 合并到 main 分支跑测试case
- main 发布 release,版本号修改
对代码的修改
功能性问题
- 请提交至少一个测试用例(Test Case)来验证对现有功能的改动。
性能相关
- 请提交必要的测试数据来证明现有代码的性能缺陷,或是新增代码的性能提升。
新功能
- 如果新增功能对已有功能不影响,请提供可以开启/关闭的开关(如 flag),并使新功能保持默认关闭的状态;
- 大型新功能(比如增加一个新的模块)开发之前,请先提交一个 issue,讨论完毕之后再进行开发。
Documentation
- https://goio.dev
How to install
go get -u github.com/gif-gif/go.io
How to use
项目启动时初始化 ,日志文件默认在项目目录下 logs/date.log
package main
import (
"fmt"
golog "github.com/gif-gif/go.io/go-log"
gomessage "github.com/gif-gif/go.io/go-message"
"github.com/gif-gif/go.io/goio"
)
type Config struct {
Name string
FeiShu string
Mode string
}
func main() {
c := Config{}
goio.Init(goio.Environment(c.Mode))
//golog.SetAdapter(golog.NewFileAdapter()) //当前工程目录logs/date.log, 可通过这个设置改变日志输出目录
//不设置 golog.SetAdapter 默认控制台输出
golog.WithHook(func(msg *golog.Message) {
if msg.Level > golog.ERROR { //致命错误以上
gomessage.FeiShu(c.FeiShu, fmt.Sprintf(">> %s/%s >> %s",
c.Name, c.Mode, string(msg.JSON())))
}
})
// or
//goio.Init(goio.Environment(c.Mode))
//goio.SetupLogDefault()
//goio.Setup(c.FeiShu)
}
发送通知
飞书
普通群消息
gomessage.FeiShu(hookUrl, "这是普通的群消息")
钉钉
gomessage.InitDingDing("token","secret")
普通群消息
err := gomessage.DingDing("这是普通的群消息")
@特定人的消息
@对象必须为绑定钉钉的手机号
err := gomessage.DingDing("Lucy, Harvey, 你们的程序挂了", "18578924567", "+13414567890")
@所有人的消息
err := gomessage.DingDing("这是@所有人的消息", "*")
Telegram电报
gomessage.InitTelegram("token",false)
//chatId 个人ID或群组ID text 消息内容
err := gomessage.TelegramTo(chatId, "text")
go-event 基于 chan
观察者模式 事件中心
//使用方法
package main
import (
goevent "github.com/gif-gif/go.io/go-event"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"time"
)
func main() {
goio.Init()
event := goevent.New()
event.Subscribe("test", func(msg goevent.Message) {
golog.WithTag("goevent").Info(msg)
})
event.Publish("test", "test")
time.Sleep(time.Duration(1) * time.Second)
}
GoHttp 模块
package main
import (
"context"
"fmt"
gocontext "github.com/gif-gif/go.io/go-context"
"github.com/gif-gif/go.io/go-http"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"time"
)
func main() {
goio.Init(goio.DEVELOPMENT)
type httpRequest struct {
Email string `json:"email"`
}
req := &gohttp.Request{
Url: "/main",
Urls: []string{ //Retry urls
"/main1",
"/main2",
"/main3",
},
QueryParams: map[string]string{"name": "jk"},
Timeout: time.Second * 2,
Body: &httpRequest{
Email: "[email protected]",
},
}
gh := &gohttp.GoHttp[gohttp.Response]{
Request: req,
BaseUrl: "http://localhost",
Headers: map[string]string{
"User-Agent": "github.com/gif-gif/go.io",
},
}
rst, err := gh.HttpPostJson(context.Background())
if err != nil {
golog.WithTag("http").Error(err.Error())
} else {
fmt.Println(rst)
}
}
验证码(支持分布式验证,基于redis)
config := goredisc.Config{
Name: "gocaptcha",
Addrs: []string{"127.0.0.1:6379"},
Password: "",
DB: 0,
Prefix: "gocaptcha",
AutoPing: true,
}
err := gocaptcha.Init(gocaptcha.Config{
RedisConfig: &config,
})
if err != nil {
golog.Error(err.Error())
return
}
data, err := gocaptcha.Default().DigitCaptcha(0, 0, 0)
golog.WithTag("data").Info(data)
gojob 模块
func simpleUseGoJob() {
n := 0
cron, err := gojob.New()
if err != nil {
golog.WithTag("gojob").Error(err)
}
defer cron.Stop()
cron.Start()
job, err := cron.SecondX(nil, 1, func(nn int) error {
golog.WithTag("gojob").Info("testing->" + gconv.String(nn))
return nil
}, n)
if err != nil {
golog.WithTag("gojob").Error(err)
} else {
golog.WithTag("gojob").Info("job.ID:" + job.ID().String())
}
time.Sleep(time.Second * 500)
golog.InfoF("end of gojob")
}
godb 模块
- mysql
- sqlite3
- clickhouse
- postgresql
- sqlserver
- tidb
func mysqlTest() {
err := gogorm.Init(gogorm.Config{
DataSource: "root:223238@tcp(127.0.0.1:33060)/gromdb?charset=utf8mb4&parseTime=True&loc=Localb",
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
// Create
insertProduct := &Product{Code: "D42", Price: 100}
db.Create(insertProduct)
fmt.Println(insertProduct.ID)
// Read
var product Product
tx := db.First(&product, 1) // find product with integer primary key
if tx.Error != nil {
fmt.Println("not found first ", tx.Error.Error())
}
db.First(&product, "code = ?", "D42")
// Delete - delete product
db.Delete(&product, 1)
}
- 其他各个模块功能在go-[模块]目录中readme或者testCase中找到使用方法
Thanks
- https://github.com/IBM/sarama
- https://gorm.io/gorm
- https://github.com/redis/go-redis
- https://github.com/aliyun/aliyun-oss-go-sdk
- https://github.com/go-co-op/gocron
- https://github.com/minio/minio-go
- https://github.com/oschwald/geoip2-golang
- https://github.com/ip2location/ip2location-go
- https://github.com/wechatpay-apiv3/wechatpay-go
- https://github.com/smartwalle/alipay
- https://github.com/360EntSecGroup-Skylar/excelize
- https://github.com/gin-gonic/gin
- https://github.com/go-resty/resty
- https://github.com/olivere/elastic
- https://github.com/elastic/go-elasticsearch
- https://github.com/mongodb/mongo-go-driver
- https://github.com/panjf2000/ants
- https://github.com/tucnak/telebot
- https://github.com/PaulSonOfLars/gotgbot
- https://github.com/mojocn/base64Captcha
- https://github.com/wenlng/go-captcha
- https://github.com/mochi-mqtt/server
- https://github.com/eclipse/paho.mqtt.golang
- https://github.com/apache/rocketmq-client-go
- https://github.com/hibiken/asynq
- https://github.com/xuri/excelize
- https://github.com/samber/lo
- https://github.com/shopspring/decimal
- https://github.com/liqiongtao/googo.io
go-apk/README.md (152 B)
go-apk
分析apk内部信息
包名:
版本名称:
版本号:
apkmd5值:
app名称:
cpu架构:
原始签名:
md5小写签名:
go-cache/cache.go (722 B)
package gocache
import (
"context"
"github.com/patrickmn/go-cache"
"time"
)
type GoCache struct {
SharedCache *cache.Cache
ctx context.Context
Config Config
}
func New(conf Config) (cli *GoCache) {
cli = &GoCache{ctx: context.TODO(), Config: conf}
cli.SharedCache = cache.New(time.Second*time.Duration(conf.DefaultExpiration), time.Duration(conf.CleanupInterval)*time.Second)
return
}
func (cli *GoCache) Get(key string) (interface{}, bool) {
if cli.SharedCache == nil {
return nil, false
}
return cli.SharedCache.Get(key)
}
func (cli *GoCache) Set(key string, data interface{}, duration time.Duration) {
if cli.SharedCache == nil {
return
}
cli.SharedCache.Set(key, data, duration)
}
go-cache/client.go (883 B)
package gocache
import (
"errors"
)
var __clients = map[string]*GoCache{}
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name] = New(conf)
}
return
}
func GetClient(names ...string) *GoCache {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoCache {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
return nil
}
go-cache/config.go (287 B)
package gocache
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
DefaultExpiration int64 `yaml:"DefaultExpiration" json:"defaultExpiration,optional"` //秒
CleanupInterval int64 `yaml:"CleanupInterval" json:"cleanupInterval,optional"` //秒
}
go-captcha/client.go (1.1 KiB)
package gocaptcha
import (
"errors"
)
var __clients = map[string]*GoCaptcha{}
// 可以一次初始化多个实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
if conf.RedisConfig == nil {
__clients[name] = NewDefault()
} else {
__clients[name], err = NewRedis(*conf.RedisConfig)
}
if err != nil {
return
}
}
return
}
func GetClient(names ...string) *GoCaptcha {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoCaptcha {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
return nil
}
go-captcha/gocaptcha.go (5.3 KiB)
package gocaptcha
import (
"context"
"fmt"
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
"github.com/mojocn/base64Captcha"
"time"
)
// 默认内存,分布式用redis等
type Config struct {
Name string `json:"name,optional" yaml:"Name"`
RedisConfig *goredisc.Config `json:"redisConfig,optional" yaml:"RedisConfig"`
}
type configJsonBody struct {
Id string
CaptchaType string
VerifyValue string
DriverAudio *base64Captcha.DriverAudio
DriverString *base64Captcha.DriverString
DriverChinese *base64Captcha.DriverChinese
DriverMath *base64Captcha.DriverMath
DriverDigit *base64Captcha.DriverDigit
}
type CaptchaData struct {
Data string `json:"data"`
CaptchaId string `json:"captchaId"`
Answer string `json:"answer"`
}
type GoCaptcha struct {
store base64Captcha.Store //验证码信息自定义存储
}
type RedisStore struct {
redis *goredisc.GoRedisC
Context context.Context
}
func (r *RedisStore) Set(id string, value string) error {
r.redis.SetEx(id, value, 10*time.Minute)
return nil
}
func (r *RedisStore) Get(id string, clear bool) string {
rst := r.redis.Get(id).Val()
if clear {
r.redis.Del(id)
}
return rst
}
func (r *RedisStore) Verify(id, answer string, clear bool) bool {
rst := r.Get(id, clear)
return rst == answer
}
// new other store
func NewRedis(config goredisc.Config) (*GoCaptcha, error) {
err := goredisc.Init(config)
if err != nil {
return nil, err
}
redis := goredisc.GetClient(config.Name)
if redis == nil {
return nil, fmt.Errorf("failed to connect to redis %v", config)
}
return New(&RedisStore{
redis: redis,
Context: context.Background(),
}), nil
}
func New(store base64Captcha.Store) *GoCaptcha {
return &GoCaptcha{
store: store,
}
}
func NewDefault() *GoCaptcha {
return New(base64Captcha.DefaultMemStore)
}
// 返回不同类型的验证码
func (g *GoCaptcha) GetCaptcha(param configJsonBody) (*CaptchaData, error) {
var driver base64Captcha.Driver
//create base64 encoding captcha
switch param.CaptchaType {
case "audio":
driver = param.DriverAudio
case "string":
driver = param.DriverString.ConvertFonts()
case "math":
driver = param.DriverMath.ConvertFonts()
case "chinese":
driver = param.DriverChinese.ConvertFonts()
default:
driver = param.DriverDigit
}
c := base64Captcha.NewCaptcha(driver, g.store)
id, b64s, answer, err := c.Generate()
if err != nil {
return nil, err
}
data := CaptchaData{
Data: b64s,
CaptchaId: id,
Answer: answer,
}
return &data, nil
}
// 验证
func (g *GoCaptcha) CaptchaVerify(id, code string) bool {
if id == "" || code == "" {
return false
}
return g.store.Verify(id, code, true)
}
func (g *GoCaptcha) DigitCaptcha(width, height, length int) (*CaptchaData, error) {
if width == 0 {
width = 240
}
if height == 0 {
height = 80
}
if length == 0 {
length = 4
}
var param = configJsonBody{
CaptchaType: "",
DriverDigit: &base64Captcha.DriverDigit{
Length: length,
Height: height,
Width: width,
DotCount: 2,
},
}
data, err := g.GetCaptcha(param)
if err != nil {
return nil, fmt.Errorf("DigitCaptcha errr")
}
return data, nil
}
func (g *GoCaptcha) StringCaptcha(width, height, length int) (*CaptchaData, error) {
if width == 0 {
width = 240
}
if height == 0 {
height = 80
}
var param = configJsonBody{
CaptchaType: "string",
DriverString: &base64Captcha.DriverString{
Length: length,
Height: height,
Width: width,
ShowLineOptions: 2,
NoiseCount: 0,
Source: "1234567890qwertyuioplkjhgfdsazxcvbnm",
},
}
data, err := g.GetCaptcha(param)
if err != nil {
return nil, fmt.Errorf("StringCaptcha errr")
}
return data, nil
}
func (g *GoCaptcha) AudioCaptcha(language string, length int) (*CaptchaData, error) {
if language == "" {
language = "123456"
}
var param = configJsonBody{
CaptchaType: "audio",
DriverAudio: &base64Captcha.DriverAudio{
Length: length,
Language: language,
},
}
data, err := g.GetCaptcha(param)
if err != nil {
return nil, fmt.Errorf("AudioCaptcha errr")
}
return data, nil
}
// source是中文英文字列表
func (g *GoCaptcha) ChineseCaptcha(width, height, length int, source string) (*CaptchaData, error) {
if source == "" {
source = "123456qwertyu你好adfkl在在载在饿工一ioplkjhgfdsazxcvbnm"
}
if width == 0 {
width = 240
}
if height == 0 {
height = 80
}
var param = configJsonBody{
CaptchaType: "chinese",
DriverChinese: &base64Captcha.DriverChinese{
Length: length,
Height: height,
Width: width,
ShowLineOptions: 0,
NoiseCount: 0,
Source: source,
},
}
data, err := g.GetCaptcha(param)
if err != nil {
return nil, fmt.Errorf("ChineseCaptcha errr")
}
return data, nil
}
// 数学计算
func (g *GoCaptcha) MathCaptcha(width, height int) (*CaptchaData, error) {
if width == 0 {
width = 240
}
if height == 0 {
height = 80
}
var param = configJsonBody{
CaptchaType: "math",
DriverMath: &base64Captcha.DriverMath{
Height: height,
Width: width,
ShowLineOptions: 0,
NoiseCount: 0,
},
}
data, err := g.GetCaptcha(param)
if err != nil {
return nil, fmt.Errorf("MathCaptcha errr")
}
return data, nil
}
go-captcha/readme.md (190 B)
验证码
- https://github.com/mojocn/base64Captcha (当前)支持 数字 字符串 中文 数学 语音
- https://github.com/wenlng/go-captcha (高级) 旋转图片,复杂验证码
go-captcha/test/test.go (4.8 KiB)
// example of HTTP server that uses the captcha package.
package main
import (
"context"
"encoding/json"
gocaptcha "github.com/gif-gif/go.io/go-captcha"
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gin-gonic/gin"
"github.com/mojocn/base64Captcha"
"log"
"net/http"
"os"
"os/signal"
"time"
)
// configJsonBody json request body.
type configJsonBody struct {
Id string
CaptchaType string
VerifyValue string
DriverAudio *base64Captcha.DriverAudio
DriverString *base64Captcha.DriverString
DriverChinese *base64Captcha.DriverChinese
DriverMath *base64Captcha.DriverMath
DriverDigit *base64Captcha.DriverDigit
}
var store = base64Captcha.DefaultMemStore
// base64Captcha create http handler
func generateCaptchaHandler(w http.ResponseWriter, r *http.Request) {
//parse request parameters
//decoder := json.NewDecoder(r.Body)
//var param configJsonBody
//err := decoder.Decode(¶m)
//if err != nil {
// log.Println(err)
//}
var param configJsonBody = configJsonBody{
Id: "",
CaptchaType: "audio",
VerifyValue: "",
DriverAudio: &base64Captcha.DriverAudio{
Length: 4,
Language: "123456",
},
DriverString: &base64Captcha.DriverString{
Length: 4,
Height: 60,
Width: 240,
ShowLineOptions: 2,
NoiseCount: 0,
Source: "1234567890qwertyuioplkjhgfdsazxcvbnm",
},
DriverChinese: &base64Captcha.DriverChinese{
Length: 4,
Height: 60,
Width: 240,
ShowLineOptions: 2,
NoiseCount: 0,
Source: "1234567890qwertyu你好adfkl在在载在饿工一ioplkjhgfdsazxcvbnm",
},
DriverMath: &base64Captcha.DriverMath{},
DriverDigit: &base64Captcha.DriverDigit{
Length: 4,
Height: 60,
Width: 240,
DotCount: 2,
},
}
defer r.Body.Close()
var driver base64Captcha.Driver
//create base64 encoding captcha
switch param.CaptchaType {
case "audio":
driver = param.DriverAudio
case "string":
driver = param.DriverString.ConvertFonts()
case "math":
driver = param.DriverMath.ConvertFonts()
case "chinese":
driver = param.DriverChinese.ConvertFonts()
default:
driver = param.DriverDigit
}
c := base64Captcha.NewCaptcha(driver, store)
id, b64s, answer, err := c.Generate()
body := map[string]interface{}{"code": 1, "data": b64s, "captchaId": id, "answer": answer, "msg": "success"}
if err != nil {
body = map[string]interface{}{"code": 0, "msg": err.Error()}
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(body)
}
// base64Captcha verify http handler
func captchaVerifyHandle(w http.ResponseWriter, r *http.Request) {
//parse request json body
decoder := json.NewDecoder(r.Body)
var param configJsonBody
err := decoder.Decode(¶m)
if err != nil {
log.Println(err)
}
defer r.Body.Close()
//verify the captcha
body := map[string]interface{}{"code": 0, "msg": "failed"}
if store.Verify(param.Id, param.VerifyValue, true) {
body = map[string]interface{}{"code": 1, "msg": "ok"}
}
//set json response
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(body)
}
// start a net/http server
func main() {
config := goredisc.Config{
Name: "gocaptcha",
Addrs: []string{"127.0.0.1:6379"},
Password: "",
DB: 0,
Prefix: "gocaptcha",
AutoPing: true,
}
err := gocaptcha.Init(gocaptcha.Config{
RedisConfig: &config,
})
if err != nil {
golog.Error(err.Error())
return
}
data, err := gocaptcha.Default().DigitCaptcha(0, 0, 0)
golog.WithTag("data").Info(data)
simpleServer()
}
func simpleServer() {
r := gin.Default()
r.GET("/ping", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{
"message": "pong",
})
})
r.GET("/api/getCaptcha", func(c *gin.Context) {
//generateCaptchaHandler(c.Writer, c.Request)
//c.JSON(http.StatusOK, goutils.CaptchaGet(240, 60))
//c.JSON(http.StatusOK, goutils.CaptchaStringGet(240, 60))
//c.JSON(http.StatusOK, goutils.CaptchaMathGet(240, 60))
c.JSON(http.StatusOK, goutils.CaptchaAudioGet("123456"))
})
r.POST("/api/verifyCaptcha", func(c *gin.Context) {
goutils.CaptchaVerify("yCjHYaNyAJdVT8yNER6r", "111")
})
srv := &http.Server{
Addr: ":1000",
Handler: r,
}
go func() {
// 服务连接
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %s\n", err)
}
}()
// 等待中断信号以优雅地关闭服务器 (设置 5 秒的超时时间)
quit := make(chan os.Signal)
signal.Notify(quit, os.Interrupt)
<-quit
log.Println("Shutdown Server ...")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
log.Fatal("Server Shutdown:", err)
}
log.Println("Server exiting")
}
go-const/http.go (16 B)
package goconst
go-context/context.go (2.0 KiB)
package gocontext
import (
"context"
"encoding/json"
golog "github.com/gif-gif/go.io/go-log"
)
// 应用程序全局上下文
var (
__ctx context.Context
)
func init() {
__ctx = WithCancel().Context
}
func Cancel() context.Context {
return __ctx
}
// 上下文
type Context struct {
context.Context
Log *golog.Entry
v map[string]any
}
func (ctx *Context) WithLog() *Context {
ctx.Log = golog.Default().WithTag()
return ctx
}
func (ctx *Context) WithParent(parent context.Context) *Context {
ctx.Context = parent
return ctx
}
func (ctx *Context) WithValue(key string, value any) *Context {
if ctx.v == nil {
ctx.v = map[string]any{}
}
ctx.v[key] = value
return ctx
}
func (ctx *Context) Value(key string) any {
if ctx.v == nil {
ctx.v = map[string]any{}
}
if v, ok := ctx.v[key]; ok {
return v
}
return nil
}
func (ctx *Context) Values() map[string]any {
if ctx.v == nil {
ctx.v = map[string]any{}
}
return ctx.v
}
func (ctx *Context) Json() []byte {
v := ctx.Values()
b, _ := json.Marshal(&v)
return b
}
func (ctx *Context) String() string {
return string(ctx.Json())
}
func WithCancel() *Context {
//ctx, _ := context.WithCancel(context.TODO())
//sig := make(chan os.Signal)
//syscall.SIGUSR1, syscall.SIGUSR2,
//signal.Notify(sig, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGBREAK, syscall.SIGKILL)
//
//go func() {
// defer func() {
// if r := recover(); r != nil {
// golog.Error(r)
// }
// }()
//
// for ch := range sig {
// switch ch {
// case syscall.SIGUSR1: // kill -USR1
// case syscall.SIGUSR2: // kill -USR2
// case syscall.SIGHUP: // kill -1
// case syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL: // kill -9 or ctrl+c
// cancel()
// }
// }
//}()
return WithCancelx()
}
func WithLog() *Context {
return Default().WithLog()
}
func WithParent(parent context.Context) *Context {
return Default().WithParent(parent)
}
func Default() *Context {
return &Context{}
}
go-context/context_test.go (388 B)
package gocontext
import (
"fmt"
"testing"
)
func TestOsExit(t *testing.T) {
go func() {
for {
select {
case <-WithCancel().Done():
fmt.Println("---1---")
return
}
}
}()
go func() {
for {
select {
case <-WithCancel().Done():
fmt.Println("---2---")
return
}
}
}()
fmt.Println("---3---")
<-WithCancel().Done()
fmt.Println("---4---")
}
go-context/context_unix.go (723 B)
package gocontext
import (
"context"
golog "github.com/gif-gif/go.io/go-log"
"os"
"os/signal"
"syscall"
)
func WithCancelx() *Context {
ctx, cancel := context.WithCancel(context.TODO())
sig := make(chan os.Signal)
signal.Notify(sig, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL)
go func() {
defer func() {
if r := recover(); r != nil {
golog.Error(r)
}
}()
for ch := range sig {
switch ch {
case syscall.SIGUSR1: // kill -USR1
case syscall.SIGUSR2: // kill -USR2
case syscall.SIGHUP: // kill -1
case syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGKILL: // kill -9 or ctrl+c
cancel()
}
}
}()
return WithParent(ctx)
}
go-context/readme.md (299 B)
signal
syscall.SIGUSR1:kill -USR1syscall.SIGUSR2:kill -USR2syscall.SIGHUP:kill -1syscall.SIGTERM:kill TERMsyscall.SIGQUIT:kill QUITsyscall.SIGINT:ctrl + csyscall.SIGKILL:kill -9应用程序捕获不到
context
Cancel()取消执行
go-db/go-clickhouse/clickhouse.go (1.4 KiB)
package goclickhouse
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoClickHouse{}
// 可以一次初始化多个Redis实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name], err = New(conf)
if err != nil {
return
}
}
return
}
func GetClient(names ...string) *GoClickHouse {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
//if l := len(names); l > 0 {
// name := names[0]
// if cli, ok := __clients[name]; ok {
// return cli
// }
// return nil
//} else {
// if l := len(__clients); l == 1 {
// for _, cli := range __clients {
// return cli
// }
// }
// return nil
//}
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoClickHouse {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("goredis").Error("no default GoClickHouse client")
return nil
}
go-db/go-clickhouse/client.go (8.1 KiB)
package goclickhouse
import (
"context"
"crypto/tls"
"database/sql"
"fmt"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
gojob "github.com/gif-gif/go.io/go-job"
golog "github.com/gif-gif/go.io/go-log"
"github.com/pkg/errors"
"time"
)
type GoClickHouse struct {
conf Config
db *sql.DB
conn driver.Conn
cron *gojob.GoJob
}
func New(conf Config) (cli *GoClickHouse, err error) {
if conf.DialTimeout == 0 {
conf.DialTimeout = 60
}
if conf.MaxIdleConn == 0 {
conf.MaxIdleConn = 5
}
if conf.MaxOpenConn == 0 {
conf.MaxOpenConn = 10
}
if conf.ConnMaxLifetime == 0 {
conf.ConnMaxLifetime = 60 * 60
}
cli = &GoClickHouse{conf: conf}
getTls := &tls.Config{
InsecureSkipVerify: conf.InsecureSkipVerify,
}
if !conf.TLS {
getTls = nil
}
op := &clickhouse.Options{
Addr: conf.Addr,
Auth: clickhouse.Auth{
Database: conf.Database,
Username: conf.User,
Password: conf.Password,
},
Protocol: clickhouse.HTTP,
TLS: getTls,
Settings: clickhouse.Settings{
//"max_execution_time": conf.MaxExecutionTime, //60,
"max_query_size": 104857600, //100M
},
//Settings: clickhouse.Settings{
// "max_memory_usage": "10000000000", // 增加内存使用限制
// "max_bytes_before_external_group_by": "20000000000", // 增加分组操作前的字节限制
// "max_block_size": "100000", // 调整块大小
//},
DialTimeout: time.Second * time.Duration(conf.DialTimeout),
Compression: &clickhouse.Compression{
Method: clickhouse.CompressionLZ4,
},
Debug: conf.Debug,
BlockBufferSize: 10,
MaxCompressionBuffer: 10240,
ClientInfo: clickhouse.ClientInfo{ // optional, please see GoClickHouse info section in the README.md
Products: []struct {
Name string
Version string
}{
{Name: "go.io", Version: "0.1"},
},
},
ConnOpenStrategy: clickhouse.ConnOpenInOrder,
}
op.Debugf = func(format string, v ...any) {
golog.Debug(v...)
}
//if conf.Debugf == nil {
// op.Debugf = func(format string, v ...any) {
// golog.Debug(v...)
// }
//} else {
// op.Debugf = conf.Debugf
//}
cli.db = clickhouse.OpenDB(op)
conn, err := clickhouse.Open(op)
if err != nil {
return nil, err
}
cli.conn = conn
cli.db.SetMaxIdleConns(conf.MaxIdleConn)
cli.db.SetMaxOpenConns(conf.MaxOpenConn)
cli.db.SetConnMaxLifetime(time.Second * time.Duration(conf.ConnMaxLifetime))
if conf.AutoPing {
cron, err := gojob.New()
if err != nil {
return nil, err
}
cron.Start()
_, err = cron.SecondX(nil, 5, cli.ping)
if err != nil {
return nil, err
}
}
return
}
func (cli *GoClickHouse) DB() *sql.DB {
return cli.db
}
func (cli *GoClickHouse) Conn() driver.Conn {
return cli.conn
}
func (cli *GoClickHouse) Close() error {
if cli.db != nil {
err := cli.db.Close()
if err != nil {
return err
}
}
if cli.cron != nil {
err := cli.cron.Stop()
if err != nil {
return err
}
}
if cli.conn != nil {
err := cli.conn.Close()
if err != nil {
return err
}
}
return nil
}
func (cli *GoClickHouse) ping() {
if cli.db == nil {
return
}
err := cli.db.Ping()
if err == nil {
return
}
if exception, ok := err.(*clickhouse.Exception); ok {
golog.WithTag("goclickhouse").WithField("err_code", exception.Code).WithField("stack_trace", exception.StackTrace).Error(exception.Message)
return
}
golog.WithTag("goclickhouse").Error(err)
}
// OPTIMIZE
func (cli *GoClickHouse) OptimizePartition(context context.Context, tableName string, isFinal bool, partitions ...string) error {
finalStr := ""
if isFinal {
finalStr = "FINAL"
}
s := "OPTIMIZE TABLE " + tableName + " " + finalStr + ";"
if len(partitions) > 0 {
for _, partition := range partitions {
s = "OPTIMIZE TABLE " + tableName + " PARTITION '" + partition + "' " + finalStr + ";"
err := cli.conn.Exec(context, s)
if err != nil {
return err
}
}
} else {
err := cli.conn.Exec(context, s)
if err != nil {
return err
}
}
return nil
}
// 注意:
// 物化视图时需要表名特殊处理如下
//
// mvInnerTableName := ".inner_id.2f31ec0d-f667-487e-8b18-c13324c27bc7" 用于获取分组列表
//
// tableName := "`" + mvInnerTableName + "`" 用于处理具体的分区
func (cli *GoClickHouse) GetPartitions(ctx context.Context, tableName string) ([]PartitionInfo, error) {
query := `
SELECT
partition,
sum(bytes) as size
FROM system.parts
WHERE table = $1
GROUP BY partition
ORDER BY partition
`
rows, err := cli.conn.Query(ctx, query, tableName)
if err != nil {
return nil, fmt.Errorf("查询分区信息失败: %v", err)
}
defer rows.Close()
var partitions []PartitionInfo
for rows.Next() {
var p PartitionInfo
if err := rows.Scan(&p.Partition, &p.Size); err != nil {
return nil, fmt.Errorf("读取分区信息失败: %v", err)
}
partitions = append(partitions, p)
}
return partitions, nil
}
// 获取分区内的最小和最大ID
func (cli *GoClickHouse) GetPartitionIDRange(ctx context.Context, tableName, partition string) (min, max uint64, err error) {
query := fmt.Sprintf(`
SELECT
min(id) as min_id,
max(id) as max_id
FROM %s
WHERE partition = $1
`, tableName)
row := cli.conn.QueryRow(ctx, query, partition)
if err := row.Scan(&min, &max); err != nil {
return 0, 0, fmt.Errorf("获取ID范围失败: %v", err)
}
return min, max, nil
}
// 删除分区内的部分数据
func (cli *GoClickHouse) DeletePartitionData(ctx context.Context, tableName string, partition string, startID, endID uint64) error {
query := fmt.Sprintf(`
ALTER TABLE %s
DELETE WHERE partition = $1 AND id >= $2 AND id < $3
`, tableName)
if err := cli.conn.Exec(ctx, query, partition, startID, endID); err != nil {
return fmt.Errorf("删除数据失败: %v", err)
}
return nil
}
// 删除整个分区(如果分区小于50GB,直接删除整个分区 时可以直接删除,否则用 DropPartition)
func (cli *GoClickHouse) dropPartition(ctx context.Context, tableName, partition string) error {
query := fmt.Sprintf(`ALTER TABLE %s DROP PARTITION $1`, tableName)
if err := cli.conn.Exec(ctx, query, partition); err != nil {
return fmt.Errorf("删除分区失败: %v", err)
}
return nil
}
// 删除分区(支持大分区删除)
//
// 注意:
// 物化视图时需要表名特殊处理如下
//
// mvInnerTableName := ".inner_id.2f31ec0d-f667-487e-8b18-c13324c27bc7" 用于获取分组列表
//
// tableName := "`" + mvInnerTableName + "`" 用于处理具体的分区
func (cli *GoClickHouse) DropPartition(ctx context.Context, tableName string, p PartitionInfo) error {
if p.Size <= MaxSizeToDelete {
// 如果分区小于50GB,直接删除整个分区
if err := cli.dropPartition(ctx, tableName, p.Partition); err != nil {
return errors.Wrapf(err, "删除分区 %s 失败: %v", p.Partition)
}
return nil
} else {
// 如果分区大于50GB,分批删除数据
minID, maxID, err := cli.GetPartitionIDRange(ctx, tableName, p.Partition)
if err != nil {
return errors.Wrapf(err, "获取分区 %s 的ID范围失败: %v", p.Partition)
}
for startID := minID; startID < maxID; startID += BatchSizeToDelete {
endID := startID + BatchSizeToDelete
if endID > maxID {
endID = maxID
}
if err := cli.DeletePartitionData(ctx, tableName, p.Partition, startID, endID); err != nil {
return errors.Wrapf(err, "删除分区 %s 中的数据(ID范围: %d-%d)失败: %v", p.Partition, startID, endID)
}
//log.Printf("成功删除分区 %s 中的数据(ID范围: %d-%d)", p.Partition, startID, endID)
// 可选:添加一些延时避免过度占用系统资源
time.Sleep(time.Second * 1)
return nil
}
}
return nil
}
// BaseModel 注意T必须为指针类型
type BaseModel[T any] struct {
Client driver.Conn
Table string
}
// BatchInsert 注意添加字段时,先发布代码,再往数据库添加字段。不然先加字段会出现插不进去
func (m *BaseModel[T]) BatchInsert(ctx context.Context, items []T) error {
batch, err := m.Client.PrepareBatch(ctx, "INSERT INTO "+m.Table)
if err != nil {
return err
}
for i := range items {
err := batch.AppendStruct(items[i])
if err != nil {
return err
}
}
err = batch.Send()
return err
}
go-db/go-clickhouse/config.go (1.6 KiB)
package goclickhouse
const (
MaxSizeToDelete = 50 * 1024 * 1024 * 1024 // 50GB in bytes
BatchSizeToDelete = 10000000 // 每批删除的记录数
)
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
//Driver string `yaml:"Driver" json:"driver,optional"`
Addr []string `yaml:"Addr" json:"addr,optional"`
User string `yaml:"User" json:"user,optional"`
Password string `yaml:"Password" json:"password,optional"`
Database string `yaml:"Database" json:"database,optional"`
DialTimeout int32 `yaml:"DialTimeout" json:"dialTimeout,optional"` // default 30 second
MaxIdleConn int `yaml:"MaxIdleConn" json:"maxIdleConn,optional"` // default 5 second
MaxOpenConn int `yaml:"MaxOpenConn" json:"maxOpenConn,optional"` // default 10 second
MaxExecutionTime int `yaml:"MaxExecutionTime" json:"maxExecutionTime,optional"` // default 60 second
ConnMaxLifetime int `yaml:"ConnMaxLifetime" json:"connMaxLifetime,optional"` //seconds 60 * 60(1hour)
TLS bool `yaml:"TLS" json:"tls,optional"` // tls true 时都会启用https 否则http
InsecureSkipVerify bool `yaml:"InsecureSkipVerify" json:"insecureSkipVerify,optional"` // tls true 才会生效
AutoPing bool `yaml:"AutoPing" json:"autoPing,optional"`
Debug bool `yaml:"Debug" json:"debug,optional"`
}
// 获取分区信息
type PartitionInfo struct {
Partition string
Size uint64
}
go-db/go-clickhouse/readme.md (4.3 KiB)
go-clickhouse clickhouse/v2版本
尽支持 http/https database/sql interface ,即将支持 native interface
初始化
goclickhouse.Init(clickhouse.Config{
Driver: "clickhouse",
Addr: "192.168.1.100:9000",
User: "root",
Password: "123456",
Database: "test",
})
创建数据库
CREATE DATABASE IF NOT EXISTS test;
创建表
sqlstr := `
CREATE TABLE IF NOT EXISTS user
(
name String,
gender String,
birthday Date
)
ENGINE = MergeTree()
ORDER BY (name, gender)
PARTITION BY toYYYYMM(birthday)
`
if _, err := goclickhouse.DB().Exec(sqlstr); err != nil {
log.Fatal(err)
}
添加数据
func insert() {
var (
tx, _ = goclickhouse.DB().Begin()
stmt, _ = tx.Prepare(`INSERT INTO user(name, gender) VALUES(?, ?)`)
)
data := []interface{}{"", ""}
if _, err := stmt.Exec(data...); err != nil {
golog.Error(err)
return
}
if err := tx.Commit(); err != nil {
golog.Error(err)
}
}
查询数据
rows, err := DB().Query("SELECT name, gender FROM user")
if err != nil {
golog.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var (
name string
gender string
)
if err := rows.Scan(&name, &gender); err != nil {
log.Fatal(err)
}
fmt.Println(name, gender)
}
if err := rows.Err(); err != nil {
golog.Fatal(err)
}
删除表
if _, err := DB().Exec("DROP TABLE user"); err != nil {
golog.Fatal(err)
}
批量执行
package main
import (
"context"
"database/sql"
"log"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
)
// 用户数据结构
type User struct {
ID int
Name string
Age int
CreatedAt time.Time
}
func main() {
// 建立连接
conn := clickhouse.OpenDB(&clickhouse.Options{
Addr: []string{"localhost:9000"},
Auth: clickhouse.Auth{
Database: "default",
Username: "default",
Password: "",
},
MaxOpenConns: 5,
MaxIdleConns: 5,
})
defer conn.Close()
ctx := context.Background()
// 方法1:使用批处理预处理语句
{
tx, err := conn.Begin()
if err != nil {
log.Fatal(err)
}
stmt, err := tx.Prepare(`
INSERT INTO users (id, name, age, created_at)
VALUES (?, ?, ?, ?)
`)
if err != nil {
log.Fatal(err)
}
defer stmt.Close()
// 批量插入数据
users := []User{
{1, "User1", 25, time.Now()},
{2, "User2", 30, time.Now()},
{3, "User3", 35, time.Now()},
}
for _, user := range users {
_, err = stmt.Exec(
user.ID,
user.Name,
user.Age,
user.CreatedAt,
)
if err != nil {
tx.Rollback()
log.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
log.Fatal(err)
}
}
// 方法2:使用批量值语法
{
_, err := conn.Exec(ctx, `
INSERT INTO users (id, name, age, created_at)
VALUES
(?, ?, ?, ?),
(?, ?, ?, ?),
(?, ?, ?, ?)
`,
4, "User4", 40, time.Now(),
5, "User5", 45, time.Now(),
6, "User6", 50, time.Now(),
)
if err != nil {
log.Fatal(err)
}
}
// 方法3:使用原生批量插入语法
{
batch, err := conn.PrepareBatch(ctx, `
INSERT INTO users (id, name, age, created_at)
`)
if err != nil {
log.Fatal(err)
}
users := []User{
{7, "User7", 55, time.Now()},
{8, "User8", 60, time.Now()},
{9, "User9", 65, time.Now()},
}
for _, user := range users {
err := batch.Append(
user.ID,
user.Name,
user.Age,
user.CreatedAt,
)
if err != nil {
log.Fatal(err)
}
}
if err := batch.Send(); err != nil {
log.Fatal(err)
}
}
}
go-db/go-clickhouse/test/main.go (791 B)
package main
import (
"fmt"
goclickhouse2 "github.com/gif-gif/go.io/go-db/go-clickhouse"
golog "github.com/gif-gif/go.io/go-log"
"log"
)
func main() {
err := goclickhouse2.Init(goclickhouse2.Config{
Addr: []string{"122.28.113.238:111"},
User: "default",
Password: "111",
Database: "xzdsp",
InsecureSkipVerify: true,
})
if err != nil {
golog.Fatal(err)
return
}
rows, err := goclickhouse2.Default().DB().Query("SELECT oaid FROM xzdsp.clickcb limit 10")
if err != nil {
golog.Fatal(err)
return
}
defer rows.Close()
for rows.Next() {
var (
oaid string
)
if err := rows.Scan(&oaid); err != nil {
log.Fatal(err)
}
fmt.Println(oaid)
}
if err := rows.Err(); err != nil {
golog.Fatal(err)
}
}
go-db/go-es/client.go (1.5 KiB)
package goes
import (
"context"
"github.com/gif-gif/go.io/go-db/go-es/eslog"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/olivere/elastic/v7"
)
type GoEs struct {
cli *elastic.Client
ctx context.Context
}
func New(conf Config, options ...elastic.ClientOptionFunc) (cli *GoEs, err error) {
options = append(options,
// 将sniff设置为false后,便不会自动转换地址
elastic.SetSniff(false),
elastic.SetGzip(true),
elastic.SetURL(conf.Addr),
elastic.SetBasicAuth(conf.User, conf.Password),
)
if conf.EnableLog {
options = append(options, elastic.SetErrorLog(eslog.Logger{golog.ERROR}))
options = append(options, elastic.SetTraceLog(eslog.Logger{golog.DEBUG}))
options = append(options, elastic.SetInfoLog(eslog.Logger{golog.INFO}))
}
cli = &GoEs{
ctx: context.Background(),
}
cli.cli, err = elastic.NewClient(options...)
if err != nil {
golog.WithTag("go-es").WithField("host", conf.Addr).WithField("options", options).Error(err)
return
}
goutils.AsyncFunc(func() {
_, _, err := cli.cli.Ping(conf.Addr).Do(cli.ctx)
if err != nil {
golog.WithTag("go-es").WithField("host", conf.Addr).WithField("options", options).Error(err)
return
}
})
goutils.AsyncFunc(func() {
_, err := cli.cli.ElasticsearchVersion(conf.Addr)
if err != nil {
golog.WithTag("go-es").WithField("host", conf.Addr).WithField("options", options).Error(err)
return
}
})
return
}
func (cli *GoEs) Client() *elastic.Client {
return cli.cli
}
go-db/go-es/client_doc_create.go (842 B)
package goes
import (
"errors"
"github.com/olivere/elastic/v7"
)
// 文档 - 添加
// index: 索引
// id: 唯一标识
// body: json格式的数据
func (cli *GoEs) DocCreate(index, id string, body interface{}) (*elastic.IndexResponse, error) {
return cli.cli.Index().Type("_doc").
Index(index).
OpType("create").
Id(id).
BodyJson(body).
Refresh("true").
Do(cli.ctx)
}
// 文档 - 批量添加
func (cli *GoEs) DocBatchCreate(index string, data map[string]interface{}) (resp *elastic.BulkResponse, err error) {
bs := cli.cli.Bulk().Index(index).Refresh("true")
for id, doc := range data {
bs.Add(elastic.NewBulkIndexRequest().Id(id).Doc(doc)).Index(index)
}
resp, err = bs.Do(cli.ctx)
if err != nil {
return
}
if l := len(resp.Failed()); l > 0 {
err = errors.New(resp.Failed()[0].Error.Reason)
}
return
}
go-db/go-es/client_doc_create_test.go (920 B)
package goes
import (
"fmt"
"log"
"testing"
)
func TestClient_DocCreate(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
resp, err := Default().DocCreate(index, "1001", map[string]interface{}{
"name": "goio",
})
fmt.Println(resp, err)
}
func TestClient_DocBatchCreate(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
data := map[string]interface{}{
"1002": map[string]interface{}{
"name": "noname_1002",
},
"1003": map[string]interface{}{
"name": "noname_1003",
},
}
resp, err := Default().DocBatchCreate(index, data)
fmt.Println(resp, err)
}
go-db/go-es/client_doc_delete.go (1.0 KiB)
package goes
import (
"github.com/olivere/elastic/v7"
)
// 文档 - 删除 - 根据ID
func (cli *GoEs) DocDelete(index, id string) (*elastic.IndexResponse, error) {
return cli.cli.Index().Index(index).Id(id).Refresh("true").Do(cli.ctx)
}
// 文档 - 删除 - 根据条件
// 即时没有符合条件的文档,也不会报404错误
func (cli *GoEs) DocDeleteBy(index string, query elastic.Query) (total int64, err error) {
var resp *elastic.BulkIndexByScrollResponse
resp, err = cli.cli.DeleteByQuery(index).Query(query).Refresh("true").Do(cli.ctx)
if err != nil {
return
}
total = resp.Deleted
return
}
// 文档 - 批量删除
func (cli *GoEs) DocBatchDelete(index string, ids []string) (total int64, err error) {
if l := len(ids); l == 0 {
return
}
bs := cli.cli.Bulk().Index(index).Refresh("true")
for _, id := range ids {
bs.Add(elastic.NewBulkDeleteRequest().Id(id))
}
var resp *elastic.BulkResponse
resp, err = bs.Do(cli.ctx)
if err != nil {
return
}
total = int64(len(resp.Succeeded()))
return
}
go-db/go-es/client_doc_select_test.go (2.8 KiB)
package goes
import (
"log"
"reflect"
"testing"
"time"
)
var server = "http://122.228.113.231:9200"
type TTest struct {
Id int `json:"id"`
UpdatedAt int `json:"updated_at"`
}
type TTT struct {
Version string `json:"@version"`
Id int `json:"id"`
UpdatedAt int `json:"updated_at"`
Event struct {
Original TTest `json:"original"`
} `json:"event"`
Timestamp time.Time `json:"@timestamp"`
}
func TestClient_DocGet(t *testing.T) {
conf := Config{
Addr: server,
User: "elastic",
Password: "elastic",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "logstash-2024.08.29"
type User struct {
Name string `json:"name"`
}
var u TTT
if _, err := Default().DocGet(index, "p103nZEBi67xN4wCq_aA", &u); err != nil {
log.Println(err)
return
}
log.Println(u)
}
func TestClient_DocMultiGet(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
type User struct {
Name string `json:"name"`
}
var us []User
if _, err := Default().DocMultiGet(index, []string{"1001", "1002", "1003"}, &us); err != nil {
log.Println(err)
return
}
log.Println(us)
}
func TestClient_DocSearch(t *testing.T) {
conf := Config{
Addr: server,
User: "elastic",
Password: "elastic",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "logstash-2024.08.29"
type User struct {
Name string `json:"name"`
}
query := Default().DocSearchMatchQuery("updated_at", 0)
p := &Pagination{
Sort: "id",
Order: true,
Offset: 0,
Limit: 10,
}
resp, err := Default().DocSearch(index, query, p)
if err != nil {
log.Println(err)
return
}
for _, i := range resp.Each(reflect.TypeOf(TTest{})) {
log.Println(i)
}
log.Println(resp.Hits.TotalHits)
log.Println(resp.Hits.Hits)
}
func TestClient_DocSearch2(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
type User struct {
Name string `json:"name"`
}
query := Default().DocSearchBoolQuery(BoolQueryFilter,
Default().DocSearchMatchQuery("name", "goio"),
Default().DocSearchMatchQuery("name", "noname_1002"))
p := &Pagination{
Sort: "name",
Order: false,
Offset: 0,
Limit: 10,
}
resp, err := Default().DocSearch(index, query, p)
if err != nil {
log.Println(err)
return
}
for _, i := range resp.Each(reflect.TypeOf(User{})) {
log.Println(i)
}
log.Println(resp.Hits.TotalHits)
log.Println(resp.Hits.Hits)
}
go-db/go-es/client_doc_selete.go (4.2 KiB)
package goes
import (
"encoding/json"
"errors"
"github.com/olivere/elastic/v7"
)
// 文档 - 查询文档 - 根据ID
func (cli *GoEs) DocGet(index, id string, m interface{}) (resp *elastic.GetResult, err error) {
resp, err = cli.cli.Get().Index(index).Id(id).Do(cli.ctx)
if err != nil {
return
}
if resp.Error != nil {
err = errors.New(resp.Error.Reason)
return
}
if m != nil {
json.Unmarshal(resp.Source, &m)
}
return
}
// 文档 - 查询文档 - 根据ID集合
func (cli *GoEs) DocMultiGet(index string, ids []string, m interface{}) (resp *elastic.MgetResponse, err error) {
ms := cli.cli.MultiGet()
for _, id := range ids {
ms.Add(elastic.NewMultiGetItem().Index(index).Id(id))
}
resp, err = ms.Do(cli.ctx)
if err != nil {
return
}
if m != nil {
var arr []interface{}
for _, doc := range resp.Docs {
var _m interface{}
json.Unmarshal(doc.Source, &_m)
arr = append(arr, _m)
}
b, _ := json.Marshal(&arr)
json.Unmarshal(b, &m)
}
return
}
type Pagination struct {
Sort string // 排序字段
Order bool // true=升序 false=降序
Offset int
Limit int
}
// 文档 - 搜索文档 - 根据条件
func (cli *GoEs) DocSearch(index string, query elastic.Query, p *Pagination) (resp *elastic.SearchResult, err error) {
ss := cli.cli.Search().Index(index).Query(query)
if p != nil {
if v := p.Sort; v != "" {
ss.Sort(v, p.Order)
}
if v := p.Offset; v > 0 {
ss.From(v)
}
if v := p.Limit; v > 0 {
ss.Size(v)
}
}
resp, err = ss.Do(cli.ctx)
if err != nil {
return
}
if resp.Error != nil {
err = errors.New(resp.Error.Reason)
}
return
}
// 文档 - 搜索条件 - 精确匹配单个字段
func (cli *GoEs) DocSearchTermQuery(field string, value interface{}) *elastic.TermQuery {
return elastic.NewTermQuery(field, value)
}
// 文档 - 搜索条件 - 精确匹配多个字段
func (cli *GoEs) DocSearchTermsQuery(field string, values []interface{}) *elastic.TermsQuery {
return elastic.NewTermsQuery(field, values...)
}
// 文档 - 搜索条件 - 匹配查找,单字段搜索(匹配分词结果)
func (cli *GoEs) DocSearchMatchQuery(field string, value interface{}) *elastic.MatchQuery {
return elastic.NewMatchQuery(field, value)
}
// 文档 - 搜索条件 - 脚本查询
func (cli *GoEs) DocSearchScriptQuery(script *elastic.Script) *elastic.ScriptQuery {
return elastic.NewScriptQuery(script)
}
// 文档 - 搜索条件 - 范围查找
func (cli *GoEs) DocSearchRangeQuery(field string, from, to interface{}) *elastic.RangeQuery {
return elastic.NewRangeQuery(field).Gte(from).Lte(to)
}
// 文档 - 搜索条件 - 判断某个字段是否存在
func (cli *GoEs) DocSearchExistsQuery(field string) *elastic.ExistsQuery {
return elastic.NewExistsQuery(field)
}
type BoolQueryAction string
var (
BoolQueryMust BoolQueryAction = "must"
BoolQueryFilter BoolQueryAction = "filter"
BoolQueryShould BoolQueryAction = "should"
BoolQueryMustNot BoolQueryAction = "must_not"
)
// 文档 - 搜索条件 - bool 组合查找
// must 条件必须要满足,并将对分数起作用
// filter 条件必须要满足,但又不同于 must 子句,在 filter context 中执行,这意味着忽略评分,并考虑使用缓存。效率会高于 must
// should 条件应该满足。可以通过 minimum_should_match 参数指定应该满足的条件个数。如果 bool 查询包含 should 子句,并且没有 must 和 filter 子句,则默认值为 1,否则默认值为 0
// must_not 条件必须不能满足。在 filter context 中执行,这意味着评分被忽略,并考虑使用缓存。因为评分被忽略,所以会返回所有 0 分的文档
func (cli *GoEs) DocSearchBoolQuery(action BoolQueryAction, queries ...elastic.Query) *elastic.BoolQuery {
switch action {
case BoolQueryMust:
return elastic.NewBoolQuery().Must(queries...)
case BoolQueryFilter:
return elastic.NewBoolQuery().Filter(queries...)
case BoolQueryShould:
return elastic.NewBoolQuery().Should(queries...)
case BoolQueryMustNot:
return elastic.NewBoolQuery().MustNot(queries...)
}
return elastic.NewBoolQuery()
}
// 文档 - 查询文档数量
func (cli *GoEs) DocCount(index string, query elastic.Query) (int64, error) {
return cli.cli.Count(index).Query(query).Do(cli.ctx)
}
go-db/go-es/client_doc_update.go (3.7 KiB)
package goes
import (
"errors"
"fmt"
"github.com/olivere/elastic/v7"
)
// 文档 - 修改
func (cli *GoEs) DocUpdate(index, id string, body interface{}) (response *elastic.UpdateResponse, err error) {
return cli.cli.Update().
Index(index).
Id(id).
Doc(body).
Refresh("true").
Do(cli.ctx)
}
// 文档 - 修改 - 不存在就插入
func (cli *GoEs) DocUpset(index, id string, body interface{}) (response *elastic.UpdateResponse, err error) {
return cli.cli.Update().
Index(index).
Id(id).
Doc(body).
Upsert(body).
Refresh("true").
Do(cli.ctx)
}
// 文档 - 修改 - 根据条件
func (cli *GoEs) DocUpdateBy(index string, query elastic.Query, script *elastic.Script) (total int64, err error) {
var resp *elastic.BulkIndexByScrollResponse
resp, err = cli.cli.UpdateByQuery(index).Query(query).Script(script).Refresh("true").Do(cli.ctx)
if err != nil {
return
}
total = resp.Updated
return
}
// 文档 - 批量修改
func (cli *GoEs) DocBatchUpdate(index string, ids []string, docs []interface{}) (err error) {
bs := cli.cli.Bulk().Index(index).Refresh("true")
for i := range ids {
bs.Add(elastic.NewBulkUpdateRequest().Id(ids[i]).Doc(docs[i]))
}
var resp *elastic.BulkResponse
resp, err = bs.Do(cli.ctx)
if l := len(resp.Failed()); l > 0 {
err = errors.New(resp.Failed()[0].Error.Reason)
return
}
return
}
// 文档 - 批量修改 - 不存在就插入
func (cli *GoEs) DocBatchUpset(index string, ids []string, docs []interface{}) (err error) {
bs := cli.cli.Bulk().Index(index).Refresh("true")
for i := range ids {
bs.Add(elastic.NewBulkUpdateRequest().Id(ids[i]).Doc(docs[i]).Upsert(docs[i]))
}
var resp *elastic.BulkResponse
resp, err = bs.Do(cli.ctx)
if l := len(resp.Failed()); l > 0 {
err = errors.New(resp.Failed()[0].Error.Reason)
return
}
return
}
// -----------------------------------------------------------------------
// 脚本更新示例
// 参考文档:https://dablelv.blog.csdn.net/article/details/121396060
// -----------------------------------------------------------------------
// 1. 数组删除元素
func (cli *GoEs) DocArrayDelField(field string, value interface{}) *elastic.Script {
scriptStr := fmt.Sprintf(
`ctx._source.%s.remove(ctx._source.%s.indexOf(params.%s))`,
field, field, field,
)
return elastic.NewScript(scriptStr).Params(
map[string]interface{}{
field: value,
},
)
}
// 2. 数组删除多个元素
func (cli *GoEs) DocArrayDelFields(field string, value []interface{}) *elastic.Script {
scriptStr := fmt.Sprintf(
`for (int i = 0; i < params.%s.length; i++) {
if (ctx._source.%s.contains(params.%s[i])) {
ctx._source.%s.remove(ctx._source.%s.indexOf(params.%s[i]))
}
}`,
field, field, field, field, field, field,
)
return elastic.NewScript(scriptStr).Params(
map[string]interface{}{
field: value,
},
)
}
// 3. 数组追加元素
func (cli *GoEs) DocArrayAppendValue(field string, value interface{}) *elastic.Script {
scriptStr := fmt.Sprintf(
`ctx._source.%s.add(params.%s)`,
field, field,
)
return elastic.NewScript(scriptStr).Params(
map[string]interface{}{
field: value,
},
)
}
// 4. 数组追加多个元素
func (cli *GoEs) DocArrayAppendValues(field string, value []interface{}) *elastic.Script {
scriptStr := fmt.Sprintf(
`ctx._source.%s.addAll(params.%s)`,
field, field,
)
return elastic.NewScript(scriptStr).Params(
map[string]interface{}{
field: value,
},
)
}
// 5. 数组修改元素
func (cli *GoEs) DocArrayUpdateValue(field string, old, new interface{}) *elastic.Script {
scriptStr := fmt.Sprintf(
`ctx._source.%s[ctx._source.%s.indexOf(params.old)]=params.new`,
field, field,
)
return elastic.NewScript(scriptStr).Params(
map[string]interface{}{
"old": old,
"new": new,
},
)
}
go-db/go-es/client_doc_update_test.go (1.9 KiB)
package goes
import (
"fmt"
"log"
"testing"
)
func TestClient_DocUpdate(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
fmt.Println(Default().DocUpdate(index, "1001", map[string]interface{}{"name": "noname_1001"}))
}
func TestClient_DocUpset(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
fmt.Println(Default().DocUpset(index, "1004", map[string]interface{}{"name": "noname_1004"}))
}
func TestClient_DocBatchUpdate(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
var (
ids []string
docs []interface{}
)
for i := 1; i < 10; i++ {
ids = append(ids, fmt.Sprintf("%d", 1000+i))
docs = append(docs, map[string]interface{}{
"name": fmt.Sprintf("noname_%d", 1000+i),
})
}
fmt.Println(Default().DocBatchUpset(index, ids, docs))
}
func TestClient_DocBatchUpset(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
var (
ids []string
docs []interface{}
)
for i := 1; i < 10; i++ {
ids = append(ids, fmt.Sprintf("%d", 1000+i))
docs = append(docs, map[string]interface{}{
"name": fmt.Sprintf("goio_%d", 1000+i),
})
}
fmt.Println(Default().DocBatchUpset(index, ids, docs))
}
go-db/go-es/client_index.go (2.2 KiB)
package goes
import "github.com/olivere/elastic/v7"
// 索引 - 所有索引名称
func (cli *GoEs) IndexNames() ([]string, error) {
return cli.cli.IndexNames()
}
// 索引 - 是否存在
func (cli *GoEs) IndexExists(index string) (bool, error) {
return cli.cli.IndexExists(index).Do(cli.ctx)
}
// 索引 - 创建
func (cli *GoEs) IndexCreate(index, body string) (err error) {
var exist bool
exist, err = cli.IndexExists(index)
if err != nil {
return
}
if exist {
err = nil
return
}
_, err = cli.cli.CreateIndex(index).BodyString(body).Do(cli.ctx)
return
}
// 索引 - 查询 - 文档结构、索引设置、data
func (cli *GoEs) IndexGet(index string) (*elastic.IndicesGetResponse, error) {
resp, err := cli.cli.IndexGet().Index(index).Do(cli.ctx)
if err != nil {
return nil, err
}
return resp[index], nil
}
// 索引 - 查看 - 文档结构
func (cli *GoEs) IndexMapping(index string) (map[string]interface{}, error) {
return cli.cli.GetMapping().Index(index).Do(cli.ctx)
}
// 索引 - 修改 - 文档结构
func (cli *GoEs) IndexUpdateMapping(index, body string) (*elastic.PutMappingResponse, error) {
return cli.cli.PutMapping().Index(index).BodyString(body).Do(cli.ctx)
}
// 索引 - 查看 - 索引设置
func (cli *GoEs) IndexSettings(index string) (map[string]*elastic.IndicesGetSettingsResponse, error) {
return cli.cli.IndexGetSettings().Index(index).Do(cli.ctx)
}
// 索引 - 修改 - 索引设置
func (cli *GoEs) IndexUpdateSettings(index, body string) (*elastic.IndicesPutSettingsResponse, error) {
return cli.cli.IndexPutSettings().Index(index).BodyString(body).Do(cli.ctx)
}
// 索引 - 别名 - 添加
func (cli *GoEs) IndexAlias(index, aliasName string) (*elastic.AliasResult, error) {
return cli.cli.Alias().Add(index, aliasName).Do(cli.ctx)
}
// 索引 - 别名 - 删除
func (cli *GoEs) IndexAliasRemove(index, aliasName string) (*elastic.AliasResult, error) {
return cli.cli.Alias().Remove(index, aliasName).Do(cli.ctx)
}
// 索引 - 删除
func (cli *GoEs) IndexDelete(index string) (err error) {
var exist bool
exist, err = cli.IndexExists(index)
if err != nil {
return
}
if !exist {
err = nil
return
}
_, err = cli.cli.DeleteIndex(index).Do(cli.ctx)
return
}
go-db/go-es/client_index_test.go (946 B)
package goes
import (
"fmt"
"log"
"testing"
)
func TestClient_IndexGet(t *testing.T) {
conf := Config{
Addr: "http://192.168.1.100:9200",
User: "elastic",
Password: "123456",
EnableLog: true,
}
if err := Init(conf); err != nil {
log.Println(err.Error())
return
}
index := "test_202209"
//if err := Default().IndexCreate(index, ``); err != nil {
// fmt.Println(err)
// return
//}
//fmt.Println(Default().IndexGet(index))
//fmt.Println(Default().IndexMapping(index))
//fmt.Println(Default().IndexSettings(index))
fmt.Println(Default().IndexUpdateMapping(index, `{"properties": {"name": {"type":"text", "fielddata": true}}}`))
//fmt.Println(Default().IndexExists(index))
//fmt.Println(Default().IndexNames())
//fmt.Println(Default().IndexAlias(index, index+"_aa"))
//fmt.Println(Default().IndexAliasRemove(index, index+"_aa"))
//fmt.Println(Default().Client().ElasticsearchVersion(conf.Addr))
}
go-db/go-es/config.go (325 B)
package goes
type Config struct {
Name string `json:"name,optional" yaml:"Name"`
Addr string `json:"addr,optional" yaml:"Addr"`
User string `json:"user,optional" yaml:"User"`
Password string `json:"password,optional" yaml:"Password"`
EnableLog bool `json:"enableLog,optional" yaml:"EnableLog"`
}
go-db/go-es/es.go (984 B)
package goes
import (
"errors"
"github.com/olivere/elastic/v7"
)
var __clients = map[string]*GoEs{}
// 可以多次调用初始化多个实例
func Init(conf Config, options ...elastic.ClientOptionFunc) (err error) {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("GoEs already exists")
}
__clients[name], err = New(conf, options...)
if err != nil {
return
}
return nil
}
func GetClient(names ...string) *GoEs {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoEs {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
return nil
}
go-db/go-es/eslog/log.go (492 B)
package eslog
import (
golog "github.com/gif-gif/go.io/go-log"
"strings"
)
type Logger struct {
Level golog.Level
}
func (l Logger) Printf(format string, v ...interface{}) {
log := golog.WithTag("go-es")
switch l.Level {
case golog.DEBUG:
log.DebugF(format, v...)
case golog.INFO:
log.InfoF(format, v...)
case golog.WARN, golog.ERROR, golog.PANIC, golog.FATAL:
if strings.Contains(format, "warning") {
log.WarnF(format, v...)
return
}
log.ErrorF(format, v...)
}
}
go-db/go-es/readme.md (123 B)
go-es
目前:v7
https://github.com/olivere/elastic
官方:v7 v8
https://github.com/elastic/go-elasticsearch
go-db/go-esnew/client.go (3.2 KiB)
package goeso
import (
"context"
"encoding/json"
"github.com/elastic/go-elasticsearch/v8"
"github.com/elastic/go-elasticsearch/v8/esapi"
"github.com/elastic/go-elasticsearch/v8/esutil"
"github.com/gogf/gf/util/gconv"
"strings"
)
type GoEs struct {
es *elasticsearch.TypedClient
esc *elasticsearch.Client
ctx context.Context
}
func New(conf elasticsearch.Config) (cli *GoEs, err error) {
cli = &GoEs{
ctx: context.Background(),
}
es, err := elasticsearch.NewClient(conf)
if err != nil {
return nil, err
}
cli.esc = es
err = cli.newType(conf)
if err != nil {
return nil, err
}
//
//goutils.AsyncFunc(func() {
// //cli.es.Ping = func() *ping.Ping {
// // return nil
// //}
//})
return cli, nil
}
func (cli *GoEs) newType(conf elasticsearch.Config) (err error) {
es, err := elasticsearch.NewTypedClient(conf)
if err != nil {
return err
}
cli.es = es
return nil
}
func (cli *GoEs) TypeClient() *elasticsearch.TypedClient {
return cli.es
}
func (cli *GoEs) Client() *elasticsearch.Client {
return cli.esc
}
// 文档 - 添加
// index: 索引
// docId: 唯一标识
// body: json or struct
func (cli *GoEs) DocCreate(index, docId string, document interface{}) (*esapi.Response, error) {
if value, ok := document.(string); ok {
return cli.esc.Create(index, docId, strings.NewReader(value))
} else {
v, _ := json.Marshal(document)
return cli.esc.Create(index, docId, strings.NewReader(string(v)))
}
}
// Updating documents
// client.Update("my_index", "id", strings.NewReader(`{doc: { language: "Go" }}`))
func (cli *GoEs) DocUpdate(index, docId string, document interface{}) (*esapi.Response, error) {
if value, ok := document.(string); ok {
return cli.esc.Update(index, docId, strings.NewReader(value))
} else {
v, _ := json.Marshal(document)
return cli.esc.Update(index, docId, strings.NewReader(string(v)))
}
}
// 文档 - 批量添加
func (cli *GoEs) DocBatchCreate(index string, data map[string]interface{}) (err error) {
indexer, _ := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
Client: cli.esc,
Index: index,
})
for id, doc := range data {
err = indexer.Add(
context.Background(),
esutil.BulkIndexerItem{
Action: "index",
DocumentID: id,
Body: strings.NewReader(gconv.String(doc)),
})
if err != nil {
return err
}
}
err = indexer.Close(cli.ctx)
if err != nil {
return err
}
return nil
}
// 删除 - 根据IDS - Deleting an index
func (cli *GoEs) DeleteIndex(indexes []string) (*esapi.Response, error) {
return cli.esc.Indices.Delete(indexes)
}
// 文档 - 删除 - 根据ID - Deleting documents
func (cli *GoEs) DocDelete(index, id string) (*esapi.Response, error) {
return cli.esc.Delete(index, id)
}
// 文档 - 删除 - 根据条件
// 即时没有符合条件的文档,也不会报404错误
func (cli *GoEs) DocDeleteBy(index string, query string) (*esapi.Response, error) {
return cli.esc.DeleteByQuery([]string{index}, strings.NewReader(query))
}
// 文档 - 批量删除
func (cli *GoEs) DocBatchDelete(index string, ids []string) (total int64, err error) {
if l := len(ids); l == 0 {
return
}
total = 0
for _, id := range ids {
_, err = cli.DocDelete(index, id)
if err == nil {
total++
} else {
return total, err
}
}
return total, nil
}
go-db/go-esnew/client_doc_select.go (5.2 KiB)
package goeso
import (
"encoding/json"
"fmt"
"github.com/elastic/go-elasticsearch/v8/typedapi/core/search"
"github.com/elastic/go-elasticsearch/v8/typedapi/types"
)
// 文档 - 查询文档 - 根据ID
func (cli *GoEs) DocExists(index, id string) (bool, error) {
return cli.es.Exists(index, id).IsSuccess(nil)
}
// 文档 - 搜索条件 - 判断某个字段是否存在
func (cli *GoEs) DocSearchExistsQuery(field string) types.Query {
return types.Query{
Exists: &types.ExistsQuery{Field: field},
}
}
// 文档 - 查询文档 - 根据ID
func (cli *GoEs) DocGet(index, id string, m interface{}) (bool, error) {
res, err := cli.es.Get(index, id).Refresh(true).Do(cli.ctx)
if err != nil {
return false, err
}
if m != nil {
if len(res.Source_) > 0 {
err = json.Unmarshal(res.Source_, m)
if err != nil {
return false, err
}
}
}
return true, nil
}
// 文档 - 查询文档 - 根据ID集合
func (cli *GoEs) DocMultiGet(index string, ids []string, p *Pagination) (*search.Response, error) {
return cli.DocSearch(index, types.Query{
Ids: &types.IdsQuery{
Values: ids,
},
}, p)
}
// query := `{ "query": { "match_all": {} } }`
func (cli *GoEs) DocSearchAll(index string, p *Pagination) (*search.Response, error) {
return cli.DocSearch(index, types.Query{MatchAll: &types.MatchAllQuery{}}, p)
}
// 文档 - 搜索文档 - 根据条件
func (cli *GoEs) DocSearch(index string, query types.Query, p *Pagination) (*search.Response, error) {
if p == nil {
p = &Pagination{
Sort: "",
Limit: 1000,
Offset: 0,
}
}
s := cli.es.Search()
if index != "" {
s.Index(index)
}
s.Request(&search.Request{
Query: &query,
}).Size(p.Limit).From(p.Offset)
if p.Sort != "" {
s.Sort(p.Sort)
}
res, err := s.Do(cli.ctx)
if err != nil {
return nil, err
}
// 遍历所有结果
for _, hit := range res.Hits.Hits {
fmt.Printf("%s\n", hit.Source_)
}
return res, nil
}
// 文档 - 搜索条件 - 精确匹配单个字段
func (cli *GoEs) DocSearchTermQuery(field string, value any) types.Query {
return types.Query{
Term: map[string]types.TermQuery{
field: {Value: value},
},
}
}
// 文档 - 搜索条件 - 精确匹配多个字段
func (cli *GoEs) DocSearchTermsQuery(field string, values []interface{}) types.Query {
t := types.NewTermsQuery()
t.TermsQuery = make(map[string]types.TermsQueryField)
t.TermsQuery[field] = values
return types.Query{
Terms: t,
}
}
// 文档 - 搜索条件 - 匹配查找,单字段搜索(匹配分词结果)
func (cli *GoEs) DocSearchMatchQuery(field string, value string) types.Query {
return types.Query{
Match: map[string]types.MatchQuery{field: types.MatchQuery{Query: value}},
}
}
// 文档 - 搜索条件 - 脚本查询
func (cli *GoEs) DocSearchScriptQuery(script types.Script) types.Query {
return types.Query{
Script: &types.ScriptQuery{
Script: script,
},
}
}
func (cli *GoEs) DocSearchBoolQuery(terms map[string]interface{}) types.Query {
ts := map[string]types.TermQuery{}
for k, v := range terms {
ts[k] = types.TermQuery{Value: v}
}
return types.Query{
Bool: &types.BoolQuery{
Filter: []types.Query{
{Term: ts},
},
},
}
}
// 文档 - 查询文档数量
//func (cli *GoEs) DocCount(index string, query elastic.Query) (int64, error) {
// return cli.cli.Count(index).Query(query).Do(cli.ctx)
//}
// 文档 - 搜索条件 - 范围查找
//func (cli *GoEs) DocSearchRangeQuery(field string, from, to interface{}) *elastic.RangeQuery {
// //return elastic.NewRangeQuery(field).Gte(from).Lte(to)
//
// return types.Query{
// Ra: elastic.NewRangeQuery(field).From(from).To(to),
// }
//
// elastic.NewRangeQuery(field).From(from).To(to)
//}
//
//type BoolQueryAction string
//
//var (
// BoolQueryMust BoolQueryAction = "must"
// BoolQueryFilter BoolQueryAction = "filter"
// BoolQueryShould BoolQueryAction = "should"
// BoolQueryMustNot BoolQueryAction = "must_not"
//)
//
//// 文档 - 搜索条件 - bool 组合查找
//// must 条件必须要满足,并将对分数起作用
//// filter 条件必须要满足,但又不同于 must 子句,在 filter context 中执行,这意味着忽略评分,并考虑使用缓存。效率会高于 must
//// should 条件应该满足。可以通过 minimum_should_match 参数指定应该满足的条件个数。如果 bool 查询包含 should 子句,并且没有 must 和 filter 子句,则默认值为 1,否则默认值为 0
//// must_not 条件必须不能满足。在 filter context 中执行,这意味着评分被忽略,并考虑使用缓存。因为评分被忽略,所以会返回所有 0 分的文档
//func (cli *client) DocSearchBoolQuery(action BoolQueryAction, queries ...elastic.Query) *elastic.BoolQuery {
// switch action {
// case BoolQueryMust:
// return elastic.NewBoolQuery().Must(queries...)
//
// case BoolQueryFilter:
// return elastic.NewBoolQuery().Filter(queries...)
//
// case BoolQueryShould:
// return elastic.NewBoolQuery().Should(queries...)
//
// case BoolQueryMustNot:
// return elastic.NewBoolQuery().MustNot(queries...)
// }
//
// return elastic.NewBoolQuery()
//}
//
//// 文档 - 查询文档数量
//func (cli *client) DocCount(index string, query elastic.Query) (int64, error) {
// return cli.cli.Count(index).Query(query).Do(cli.ctx)
//}
go-db/go-esnew/readme.md (79 B)
go es 难用 :( 废弃
- 基于官方 github.com/elastic/go-elasticsearch/v8
go-db/go-esnew/test/test.go (2.5 KiB)
package main
import (
"github.com/elastic/go-elasticsearch/v8"
goes "github.com/gif-gif/go.io/go-db/go-esnew"
golog "github.com/gif-gif/go.io/go-log"
)
type document struct {
Name string `json:"name"`
}
func main() {
testSearch()
golog.Info("finish...")
}
func testSearch() {
//es, err := goes.New(elasticsearch.Config{
// Addresses: []string{"http://122.228.113.238:9200"},
// Username: "es",
// Password: "123456",
// RetryOnStatus: []int{502, 503, 504, 429},
//})
//
//if err != nil {
// golog.Error("Error creating elasticsearch:" + err.Error())
// return
//}
//
//doc := document{}
//index := "index_test"
//query := `{ "query": { "match_all": {} } }`
//
//_, err = es.DocSearch(index, types.Query{MatchAll: &types.MatchAllQuery{}}, nil)
//res, err := es.DocMultiGet(index, []string{"1", "2"})
//if err != nil {
// return
//}
//遍历所有结果
//for _, hit := range res.Hits.Hits {
// fmt.Printf("%s\n", hit.Source_)
//}
//
//_, err = es.DocCreate(index, "2", str)
//if err != nil {
// golog.Error("Error DocCreate elasticsearch:" + err.Error())
// return
//}
//
//_, err = es.DeleteIndex([]string{"index1"})
//if err != nil {
// golog.Error("Error deleting elasticsearch:" + err.Error())
// return
//}
}
func testCreateDoc() {
es, err := goes.New(elasticsearch.Config{
Addresses: []string{"http://122.228.113.238:9200"},
Username: "es",
Password: "123456",
RetryOnStatus: []int{502, 503, 504, 429},
})
if err != nil {
golog.Error("Error creating elasticsearch:" + err.Error())
return
}
doc := document{Name: "test"}
index := "test"
_, err = es.DocCreate(index, "2", doc)
if err != nil {
golog.Error("Error DocCreate elasticsearch:" + err.Error())
return
}
//_, err = es.DeleteIndex([]string{"index1"})
//if err != nil {
// golog.Error("Error deleting elasticsearch:" + err.Error())
// return
//}
golog.Info("finish...")
}
func testDocGet() {
es, err := goes.New(elasticsearch.Config{
Addresses: []string{"http://122.228.113.238:9200"},
Username: "es",
Password: "123456",
RetryOnStatus: []int{502, 503, 504, 429},
})
if err != nil {
golog.Error("Error creating elasticsearch:" + err.Error())
return
}
type document struct {
Name string `json:"name"`
}
doc := document{}
index := "index_test"
_, err = es.DocGet(index, "1", &doc)
if err != nil {
golog.Error("Error DocGet elasticsearch:" + err.Error())
return
} else {
golog.Info(doc.Name)
}
}
go-db/go-esnew/types.go (146 B)
package goeso
type Pagination struct {
Sort string // 排序字段 "example --> published_at:desc "
Offset int
Limit int // default 1000
}
go-db/go-mongo/client.go (1.3 KiB)
package gomongo
import (
"context"
"fmt"
gojob "github.com/gif-gif/go.io/go-job"
golog "github.com/gif-gif/go.io/go-log"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
)
//var MongoDSN = "mongodb://root:[email protected]:27017"
type GoMongo struct {
*mongo.Client
conf Config
ctx context.Context
}
func New(conf Config) (cli *GoMongo, err error) {
cli = &GoMongo{conf: conf, ctx: context.TODO()}
var uri string
if conf.EnablePassword {
uri = fmt.Sprintf("mongodb://%s:%s@%s", conf.User, conf.Password, conf.Addr)
} else {
uri = fmt.Sprintf("mongodb://%s", conf.Addr)
}
opts := options.Client().ApplyURI(uri)
cli.Client, err = mongo.Connect(cli.ctx, opts)
if err != nil {
golog.WithTag("gomongo").Error(err)
return
}
if err = cli.Ping(cli.ctx, readpref.Primary()); err != nil {
golog.WithTag("gomongo").Error(err)
return
}
if conf.AutoPing {
gj, _ := gojob.New()
gj.Start()
gj.SecondX(nil, 5, func() {
if err := cli.Ping(cli.ctx, readpref.Primary()); err != nil {
golog.WithTag("gomongo").Error(err)
}
})
}
return
}
func (cli *GoMongo) WithContext(ctx context.Context) *GoMongo {
cli.ctx = ctx
return cli
}
func (cli *GoMongo) DB() *mongo.Database {
return cli.Database(cli.conf.Database)
}
go-db/go-mongo/config.go (490 B)
package gomongo
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
Addr string `yaml:"Addr" json:"addr,optional"`
User string `yaml:"User" json:"user,optional"`
Password string `yaml:"Password" json:"password,optional"`
EnablePassword bool `yaml:"EnablePassword" json:"enablePassword,optional"`
Database string `yaml:"Database" json:"database,optional"`
AutoPing bool `yaml:"AutoPing" json:"autoPing,optional"`
}
go-db/go-mongo/mongo.go (1.1 KiB)
package gomongo
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoMongo{}
// 可以一次初始化多个Mongo实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name], err = New(conf)
if err != nil {
return
}
}
return
}
func GetClient(names ...string) *GoMongo {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoMongo {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("gomongo").Error("no default mongo client")
return nil
}
go-db/go-mongo/readme.md (16 B)
golang mongodb
go-db/go-mongo/test/main.go (966 B)
package main
import (
"context"
gomongo "github.com/gif-gif/go.io/go-db/go-mongo"
golog "github.com/gif-gif/go.io/go-log"
"go.mongodb.org/mongo-driver/bson"
"log"
"time"
)
func main() {
config := gomongo.Config{
Name: "",
Addr: "127.0.0.1:27017",
User: "",
Password: "",
Database: "",
AutoPing: true,
}
err := gomongo.Init(config)
if err != nil {
golog.ErrorF("error:%v", err)
return
}
client := gomongo.Default()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
collection := client.DB().Collection("users")
cur, err := collection.Find(ctx, bson.D{})
if err != nil {
log.Fatal(err)
}
defer cur.Close(ctx)
for cur.Next(ctx) {
var result bson.D
err := cur.Decode(&result)
if err != nil {
log.Fatal(err)
}
// do something with result....
}
if err := cur.Err(); err != nil {
log.Fatal(err)
}
time.Sleep(time.Second * 500)
golog.InfoF("end of gomongo")
}
go-db/go-redis/README.md (702 B)
go-redis
redis 连接和操作 基于 github.com/go-redis/redis 库
- 建议优先使用 goredisc (兼容集群和单点)
- 使用方法
config := goredisc.Config{
Name: "goredis",
Addrs: []string{"127.0.0.1:6379"},
Password: "",
DB: 0,
Prefix: "goredis",
AutoPing: true,
}
err := goredisc.Init(config)
if err != nil {
golog.WithTag("goredis").Error(err)
}
cmd := goredisc.Default().Set("goredis", "goredis")
if cmd.Err() != nil {
golog.WithTag("goredis").Error(cmd.Err())
}
v := goredisc.Default().Get("goredis").Val()
golog.WithTag("goredis").InfoF(v)
go-db/go-redis/client.go (18.7 KiB)
package goredis
import (
"context"
gojob "github.com/gif-gif/go.io/go-job"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gogf/gf/container/garray"
"github.com/redis/go-redis/v9"
"github.com/samber/lo"
"time"
)
type GoRedis struct {
Redis *redis.Client
Config Config
Ctx context.Context
}
func New(conf Config) (cli *GoRedis, err error) {
if conf.Type == "" {
conf.Type = "node"
}
cli = &GoRedis{
Ctx: context.Background(),
Config: conf,
}
cli.Redis = redis.NewClient(&redis.Options{
Addr: conf.Addr,
Password: conf.Password,
DB: conf.DB,
PoolSize: lo.If(conf.PoolSize == 0, 10).Else(conf.PoolSize),
DialTimeout: lo.If(conf.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.DialTimeout) * time.Second),
ReadTimeout: lo.If(conf.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.ReadTimeout) * time.Second),
WriteTimeout: lo.If(conf.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.WriteTimeout) * time.Second),
})
cli.Redis.AddHook(&RedisHook{Prefix: conf.Prefix})
ctx := context.Background()
if err = cli.Redis.Ping(ctx).Err(); err != nil {
golog.WithTag("goredis").Error(err)
return
}
if conf.AutoPing {
gj, _ := gojob.New()
gj.Start()
gj.SecondX(nil, 5, func() {
if err := cli.Redis.Ping(ctx).Err(); err != nil {
golog.WithTag("goredis").Warn("redis ping error:", err)
}
})
}
return cli, nil
}
func (s *GoRedis) WrapKey(key string) string {
if s.Config.Prefix == "" {
return key
}
return s.Config.Prefix + ":" + key
}
func (s *GoRedis) WrapKeys(keys ...string) []string {
arr := garray.NewStrArrayFromCopy(keys)
return arr.Walk(func(val string) string { return s.WrapKey(val) }).Slice()
}
// BitCount is redis bitcount command implementation.
func (s *GoRedis) BitCount(key string, start, end int64) *redis.IntCmd {
return s.Redis.BitCount(s.Ctx, s.WrapKey(key), &redis.BitCount{Start: start, End: end})
}
// BitOpAnd is redis bit operation (and) command implementation.
func (s *GoRedis) BitOpAnd(destKey string, keys ...string) *redis.IntCmd {
return s.Redis.BitOpAnd(s.Ctx, s.WrapKey(destKey), s.WrapKeys(keys...)...)
}
// BitOpNot is redis bit operation (not) command implementation.
func (s *GoRedis) BitOpNot(destKey, key string) *redis.IntCmd {
return s.Redis.BitOpNot(s.Ctx, s.WrapKey(destKey), s.WrapKey(key))
}
// BitOpOr is redis bit operation (or) command implementation.
func (s *GoRedis) BitOpOr(destKey string, keys ...string) *redis.IntCmd {
return s.Redis.BitOpOr(s.Ctx, s.WrapKey(destKey), s.WrapKeys(keys...)...)
}
// BitOpXor is redis bit operation (xor) command implementation.
func (s *GoRedis) BitOpXor(destKey string, keys ...string) *redis.IntCmd {
return s.Redis.BitOpXor(s.Ctx, s.WrapKey(destKey), s.WrapKeys(keys...)...)
}
// BitPos is redis bitpos command implementation.
func (s *GoRedis) BitPos(key string, bit, start, end int64) *redis.IntCmd {
return s.Redis.BitPos(s.Ctx, s.WrapKey(key), bit, start, end)
}
// Blpop uses passed in redis connection to execute blocking queries.
// Doesn't benefit from pooling redis connections of blocking queries
func (s *GoRedis) BLPop(timeout time.Duration, key string) *redis.StringSliceCmd {
return s.Redis.BLPop(s.Ctx, timeout, s.WrapKey(key))
}
// Del deletes keys.
func (s *GoRedis) Del(keys ...string) *redis.IntCmd {
return s.Redis.Del(s.Ctx, s.WrapKeys(keys...)...)
}
// Eval is the implementation of redis eval command.
func (s *GoRedis) Eval(script string, keys []string, args ...interface{}) *redis.Cmd {
return s.Redis.Eval(s.Ctx, script, s.WrapKeys(keys...), args...)
}
// EvalSha is the implementation of redis evalsha command.
func (s *GoRedis) EvalSha(sha string, keys []string, args ...interface{}) *redis.Cmd {
return s.Redis.EvalSha(s.Ctx, sha, s.WrapKeys(keys...), args...)
}
// Exists is the implementation of redis exists command.
func (s *GoRedis) Exists(key string) *redis.IntCmd {
return s.Redis.Exists(s.Ctx, s.WrapKey(key))
}
// Expire is the implementation of redis expire command.
func (s *GoRedis) Expire(key string, seconds int) *redis.BoolCmd {
return s.Redis.Expire(s.Ctx, s.WrapKey(key), time.Duration(seconds)*time.Second)
}
// Expireat is the implementation of redis expireat command.
func (s *GoRedis) ExpireAt(key string, time time.Time) *redis.BoolCmd {
return s.Redis.ExpireAt(s.Ctx, s.WrapKey(key), time)
}
// GeoAdd is the implementation of redis geoadd command.
func (s *GoRedis) GeoAdd(key string, geoLocation ...*redis.GeoLocation) *redis.IntCmd {
return s.Redis.GeoAdd(s.Ctx, s.WrapKey(key), geoLocation...)
}
// GeoDist is the implementation of redis geodist command.
func (s *GoRedis) GeoDist(key, member1, member2, unit string) *redis.FloatCmd {
return s.Redis.GeoDist(s.Ctx, s.WrapKey(key), member1, member2, unit)
}
// GeoHash is the implementation of redis geohash command.
func (s *GoRedis) GeoHash(key string, members ...string) *redis.StringSliceCmd {
return s.Redis.GeoHash(s.Ctx, s.WrapKey(key), members...)
}
// GeoRadius is the implementation of redis georadius command.
func (s *GoRedis) GeoRadius(key string, longitude, latitude float64, query *redis.GeoRadiusQuery) *redis.GeoLocationCmd {
return s.Redis.GeoRadius(s.Ctx, s.WrapKey(key), longitude, latitude, query)
}
// GeoRadiusByMember is the implementation of redis georadiusbymember command.
func (s *GoRedis) GeoRadiusByMember(key, member string, query *redis.GeoRadiusQuery) *redis.GeoLocationCmd {
return s.Redis.GeoRadiusByMember(s.Ctx, s.WrapKey(key), member, query)
}
// GeoPos is the implementation of redis geopos command.
func (s *GoRedis) GeoPos(key string, members ...string) *redis.GeoPosCmd {
return s.Redis.GeoPos(s.Ctx, s.WrapKey(key), members...)
}
// Get is the implementation of redis get command.
func (s *GoRedis) Get(key string) *redis.StringCmd {
return s.Redis.Get(s.Ctx, s.WrapKey(key))
}
// GetBit is the implementation of redis getbit command.
func (s *GoRedis) GetBit(key string, offset int64) *redis.IntCmd {
return s.Redis.GetBit(s.Ctx, s.WrapKey(key), offset)
}
// Hdel is the implementation of redis hdel command.
func (s *GoRedis) HDel(key string, fields ...string) *redis.IntCmd {
return s.Redis.HDel(s.Ctx, s.WrapKey(key), fields...)
}
// Hexists is the implementation of redis hexists command.
func (s *GoRedis) HExists(key, field string) *redis.BoolCmd {
return s.Redis.HExists(s.Ctx, s.WrapKey(key), field)
}
// Hget is the implementation of redis hget command.
func (s *GoRedis) HGet(key, field string) *redis.StringCmd {
return s.Redis.HGet(s.Ctx, s.WrapKey(key), field)
}
// Hgetall is the implementation of redis hgetall command.
func (s *GoRedis) HGetAll(key string) *redis.MapStringStringCmd {
return s.Redis.HGetAll(s.Ctx, s.WrapKey(key))
}
// Hincrby is the implementation of redis hincrby command.
func (s *GoRedis) HIncrBy(key, field string, increment int64) *redis.IntCmd {
return s.Redis.HIncrBy(s.Ctx, s.WrapKey(key), field, increment)
}
// Hkeys is the implementation of redis hkeys command.
func (s *GoRedis) HKeys(key string) *redis.StringSliceCmd {
return s.Redis.HKeys(s.Ctx, s.WrapKey(key))
}
// Hlen is the implementation of redis hlen command.
func (s *GoRedis) HLen(key string) *redis.IntCmd {
return s.Redis.HLen(s.Ctx, s.WrapKey(key))
}
// Hmget is the implementation of redis hmget command.
func (s *GoRedis) HMGet(key string, fields ...string) *redis.SliceCmd {
return s.Redis.HMGet(s.Ctx, s.WrapKey(key), fields...)
}
// Hset is the implementation of redis hset command.
func (s *GoRedis) HSet(key, field, value string) *redis.IntCmd {
return s.Redis.HSet(s.Ctx, s.WrapKey(key), field, value)
}
// Hsetnx is the implementation of redis hsetnx command.
func (s *GoRedis) HSetNX(key, field, value string) *redis.BoolCmd {
return s.Redis.HSetNX(s.Ctx, s.WrapKey(key), field, value)
}
// Hmset is the implementation of redis hmset command.
func (s *GoRedis) HMSet(key string, fieldsAndValues map[string]string) *redis.BoolCmd {
return s.Redis.HMSet(s.Ctx, s.WrapKey(key), fieldsAndValues)
}
// Hscan is the implementation of redis hscan command.
func (s *GoRedis) HScan(key string, cursor uint64, match string, count int64) *redis.ScanCmd {
return s.Redis.HScan(s.Ctx, s.WrapKey(key), cursor, match, count)
}
// Hvals is the implementation of redis hvals command.
func (s *GoRedis) HVals(key string) *redis.StringSliceCmd {
return s.Redis.HVals(s.Ctx, s.WrapKey(key))
}
// Incr is the implementation of redis incr command.
func (s *GoRedis) Incr(key string) *redis.IntCmd {
return s.Redis.Incr(s.Ctx, s.WrapKey(key))
}
// Incrby is the implementation of redis incrby command.
func (s *GoRedis) IncrBy(key string, increment int64) *redis.IntCmd {
return s.Redis.IncrBy(s.Ctx, s.WrapKey(key), increment)
}
// Keys is the implementation of redis keys command.
func (s *GoRedis) Keys(pattern string) *redis.StringSliceCmd {
return s.Redis.Keys(s.Ctx, pattern)
}
// Llen is the implementation of redis llen command.
func (s *GoRedis) LLen(key string) *redis.IntCmd {
return s.Redis.LLen(s.Ctx, s.WrapKey(key))
}
// Lpop is the implementation of redis lpop command.
func (s *GoRedis) LPop(key string) *redis.StringCmd {
return s.Redis.LPop(s.Ctx, s.WrapKey(key))
}
// Lpush is the implementation of redis lpush command.
func (s *GoRedis) LPush(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.LPush(s.Ctx, s.WrapKey(key), values...)
}
// Lrange is the implementation of redis lrange command.
func (s *GoRedis) LRange(key string, start, stop int64) *redis.StringSliceCmd {
return s.Redis.LRange(s.Ctx, s.WrapKey(key), start, stop)
}
// Lrem is the implementation of redis lrem command.
func (s *GoRedis) LRem(key string, count int64, value string) *redis.IntCmd {
return s.Redis.LRem(s.Ctx, s.WrapKey(key), count, value)
}
// Mget is the implementation of redis mget command.
func (s *GoRedis) MGet(keys ...string) *redis.SliceCmd {
return s.Redis.MGet(s.Ctx, s.WrapKeys(keys...)...)
}
// Persist is the implementation of redis persist command.
func (s *GoRedis) Persist(key string) *redis.BoolCmd {
return s.Redis.Persist(s.Ctx, s.WrapKey(key))
}
// Pfadd is the implementation of redis pfadd command.
func (s *GoRedis) PFAdd(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.PFAdd(s.Ctx, s.WrapKey(key), values...)
}
// Pfcount is the implementation of redis pfcount command.
func (s *GoRedis) PFCount(key string) *redis.IntCmd {
return s.Redis.PFCount(s.Ctx, s.WrapKey(key))
}
// Pfmerge is the implementation of redis pfmerge command.
func (s *GoRedis) PFMerge(dest string, keys ...string) *redis.StatusCmd {
return s.Redis.PFMerge(s.Ctx, dest, s.WrapKeys(keys...)...)
}
// Ping is the implementation of redis ping command.
func (s *GoRedis) Ping() *redis.StatusCmd {
return s.Redis.Ping(s.Ctx)
}
// Pipelined lets fn to execute pipelined commands.
// fn key must call GetKey or GetKeys to add prefix.
func (s *GoRedis) Pipelined(fn func(redis.Pipeliner) error) ([]redis.Cmder, error) {
return s.Redis.Pipelined(s.Ctx, fn)
}
// Rpop is the implementation of redis rpop command.
func (s *GoRedis) RPop(key string) *redis.StringCmd {
return s.Redis.RPop(s.Ctx, s.WrapKey(key))
}
// Rpush is the implementation of redis rpush command.
func (s *GoRedis) RPush(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.RPush(s.Ctx, s.WrapKey(key), values...)
}
// Sadd is the implementation of redis sadd command.
func (s *GoRedis) SAdd(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.SAdd(s.Ctx, s.WrapKey(key), values...)
}
// Scan is the implementation of redis scan command.
func (s *GoRedis) Scan(cursor uint64, match string, count int64) *redis.ScanCmd {
return s.Redis.Scan(s.Ctx, cursor, match, count)
}
// SetBit is the implementation of redis setbit command.
func (s *GoRedis) SetBit(key string, offset int64, value int) *redis.IntCmd {
return s.Redis.SetBit(s.Ctx, s.WrapKey(key), offset, value)
}
// Sscan is the implementation of redis sscan command.
func (s *GoRedis) SScan(key string, cursor uint64, match string, count int64) *redis.ScanCmd {
return s.Redis.SScan(s.Ctx, s.WrapKey(key), cursor, match, count)
}
// Scard is the implementation of redis scard command.
func (s *GoRedis) SCard(key string) *redis.IntCmd {
return s.Redis.SCard(s.Ctx, s.WrapKey(key))
}
// ScriptLoad is the implementation of redis script load command.
func (s *GoRedis) ScriptLoad(script string) *redis.StringCmd {
return s.Redis.ScriptLoad(s.Ctx, script)
}
// Set is the implementation of redis set command.
func (s *GoRedis) Set(key, value string) *redis.StatusCmd {
return s.Redis.Set(s.Ctx, s.WrapKey(key), value, 0)
}
func (s *GoRedis) Set1(key, value string, expiration time.Duration) *redis.StatusCmd {
return s.Redis.Set(s.Ctx, s.WrapKey(key), value, expiration)
}
// Setex is the implementation of redis setex command.
func (s *GoRedis) SetEx(key, value string, expiration time.Duration) *redis.StatusCmd {
return s.Redis.SetEx(s.Ctx, s.WrapKey(key), value, expiration)
}
func (s *GoRedis) SetNX(key, value string, expiration time.Duration) *redis.BoolCmd {
return s.Redis.SetNX(s.Ctx, s.WrapKey(key), value, expiration)
}
// Sismember is the implementation of redis sismember command.
func (s *GoRedis) SIsMember(key string, value interface{}) *redis.BoolCmd {
return s.Redis.SIsMember(s.Ctx, s.WrapKey(key), value)
}
// Smembers is the implementation of redis smembers command.
func (s *GoRedis) SMembers(key string) *redis.StringSliceCmd {
return s.Redis.SMembers(s.Ctx, s.WrapKey(key))
}
// Spop is the implementation of redis spop command.
func (s *GoRedis) SPop(key string) *redis.StringCmd {
return s.Redis.SPop(s.Ctx, s.WrapKey(key))
}
// Srandmember is the implementation of redis srandmember command.
func (s *GoRedis) SRandMember(key string) *redis.StringCmd {
return s.Redis.SRandMember(s.Ctx, s.WrapKey(key))
}
// Srem is the implementation of redis srem command.
func (s *GoRedis) SRem(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.SRem(s.Ctx, s.WrapKey(key), values...)
}
// String returns the string representation of s.
func (s *GoRedis) String() string {
return s.Redis.String()
}
// Sunion is the implementation of redis sunion command.
func (s *GoRedis) SUnion(keys ...string) *redis.StringSliceCmd {
return s.Redis.SUnion(s.Ctx, s.WrapKeys(keys...)...)
}
// Sunionstore is the implementation of redis sunionstore command.
func (s *GoRedis) SUnionStore(destination string, keys ...string) *redis.IntCmd {
return s.Redis.SUnionStore(s.Ctx, destination, s.WrapKeys(keys...)...)
}
// Sdiff is the implementation of redis sdiff command.
func (s *GoRedis) SDiff(keys ...string) *redis.StringSliceCmd {
return s.Redis.SDiff(s.Ctx, s.WrapKeys(keys...)...)
}
// Sdiffstore is the implementation of redis sdiffstore command.
func (s *GoRedis) SDiffStore(destination string, keys ...string) *redis.IntCmd {
return s.Redis.SDiffStore(s.Ctx, destination, s.WrapKeys(keys...)...)
}
// Sinter is the implementation of redis sinter command.
func (s *GoRedis) SInter(keys ...string) *redis.StringSliceCmd {
return s.Redis.SInter(s.Ctx, s.WrapKeys(keys...)...)
}
// Sinterstore is the implementation of redis sinterstore command.
func (s *GoRedis) SInterStore(destination string, keys ...string) *redis.IntCmd {
return s.Redis.SInterStore(s.Ctx, destination, s.WrapKeys(keys...)...)
}
// Ttl is the implementation of redis ttl command.
func (s *GoRedis) TTL(key string) *redis.DurationCmd {
return s.Redis.TTL(s.Ctx, s.WrapKey(key))
}
// Zadd is the implementation of redis zadd command.
func (s *GoRedis) ZAdd(key string, value ...redis.Z) *redis.IntCmd {
return s.Redis.ZAdd(s.Ctx, s.WrapKey(key), value...)
}
// Zcard is the implementation of redis zcard command.
func (s *GoRedis) ZCard(key string) *redis.IntCmd {
return s.Redis.ZCard(s.Ctx, s.WrapKey(key))
}
// Zcount is the implementation of redis zcount command.
func (s *GoRedis) ZCount(key string, max, min string) *redis.IntCmd {
return s.Redis.ZCount(s.Ctx, s.WrapKey(key), min, max)
}
// Zincrby is the implementation of redis zincrby command.
func (s *GoRedis) ZIncrBy(key string, increment float64, field string) *redis.FloatCmd {
return s.Redis.ZIncrBy(s.Ctx, s.WrapKey(key), increment, field)
}
// Zscore is the implementation of redis zscore command.
func (s *GoRedis) ZScore(key, value string) *redis.FloatCmd {
return s.Redis.ZScore(s.Ctx, s.WrapKey(key), value)
}
// Zrank is the implementation of redis zrank command.
func (s *GoRedis) ZRank(key, field string) *redis.IntCmd {
return s.Redis.ZRank(s.Ctx, s.WrapKey(key), field)
}
// Zrem is the implementation of redis zrem command.
func (s *GoRedis) Zrem(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.ZRem(s.Ctx, s.WrapKey(key), values...)
}
// Zremrangebyscore is the implementation of redis zremrangebyscore command.
func (s *GoRedis) ZRemRangeByScore(key string, max, min string) *redis.IntCmd {
return s.Redis.ZRemRangeByScore(s.Ctx, s.WrapKey(key), min, max)
}
// Zremrangebyrank is the implementation of redis zremrangebyrank command.
func (s *GoRedis) Zremrangebyrank(key string, start, stop int64) *redis.IntCmd {
return s.Redis.ZRemRangeByRank(s.Ctx, s.WrapKey(key), start, stop)
}
// Zrange is the implementation of redis zrange command.
func (s *GoRedis) Zrange(key string, start, stop int64) *redis.StringSliceCmd {
return s.Redis.ZRange(s.Ctx, s.WrapKey(key), start, stop)
}
// ZrangeWithScores is the implementation of redis zrange command with scores.
func (s *GoRedis) ZRangeWithScores(key string, start, stop int64) *redis.ZSliceCmd {
return s.Redis.ZRangeWithScores(s.Ctx, s.WrapKey(key), start, stop)
}
// ZRevRangeWithScores is the implementation of redis zrevrange command with scores.
func (s *GoRedis) ZRevRangeWithScores(key string, start, stop int64) *redis.ZSliceCmd {
return s.Redis.ZRevRangeWithScores(s.Ctx, s.WrapKey(key), start, stop)
}
// ZrangebyscoreWithScores is the implementation of redis zrangebyscore command with scores.
func (s *GoRedis) ZRangeByScoreWithScores(key string, opt *redis.ZRangeBy) *redis.ZSliceCmd {
return s.Redis.ZRangeByScoreWithScores(s.Ctx, s.WrapKey(key), opt)
}
// Zrevrange is the implementation of redis zrevrange command.
func (s *GoRedis) Zrevrange(key string, start, stop int64) *redis.StringSliceCmd {
return s.Redis.ZRevRange(s.Ctx, s.WrapKey(key), start, stop)
}
// ZrevrangebyscoreWithScores is the implementation of redis zrevrangebyscore command with scores.
func (s *GoRedis) ZRevRangeByScoreWithScores(key string, opt *redis.ZRangeBy) *redis.ZSliceCmd {
return s.Redis.ZRevRangeByScoreWithScores(s.Ctx, s.WrapKey(key), opt)
}
// Zrevrank is the implementation of redis zrevrank command.
func (s *GoRedis) ZRevRank(key, field string) *redis.IntCmd {
return s.Redis.ZRevRank(s.Ctx, s.WrapKey(key), field)
}
// Zunionstore is the implementation of redis zunionstore command.
func (s *GoRedis) ZUnionStore(dest string, store *redis.ZStore) *redis.IntCmd {
return s.Redis.ZUnionStore(s.Ctx, dest, store)
}
func (s *GoRedis) Close() error {
return s.Redis.Close()
}
go-db/go-redis/config.go (1.9 KiB)
package goredis
import (
redis0 "github.com/redis/go-redis/v9"
"github.com/zeromicro/go-zero/core/stores/cache"
"github.com/zeromicro/go-zero/core/stores/redis"
)
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
//NamePrefix string `yaml:"NamePrefix" json:"namePrefix,optional"`
Addr string `yaml:"Addr" json:"addr,optional"`
Password string `yaml:"Password" json:"password,optional"`
DB int `yaml:"DB" json:"db,optional"`
Prefix string `yaml:"Prefix" json:"prefix,optional"`
AutoPing bool `yaml:"AutoPing" json:"autoPing,optional"`
TLS bool `yaml:"TLS" json:"tls,optional"`
DialTimeout int `yaml:"DialTimeout" json:"dialTimeout,optional"`
ReadTimeout int `yaml:"ReadTimeout" json:"readTimeout,optional"`
WriteTimeout int `yaml:"WriteTimeout" json:"writeTimeout,optional"`
PoolSize int `yaml:"PoolSize" json:"poolSize,optional"`
Type string `yaml:"Type" json:",default=node,options=node|cluster"`
Weight int `yaml:"Weight" json:",default=100"` //for gozero
}
type ClusterConf []Config
// 兼容go-zore
func (c ClusterConf) GetCacheConf() cache.CacheConf {
cacheConf := make([]cache.NodeConf, 0, len(c))
for _, conf := range c {
cacheConf = append(cacheConf, cache.NodeConf{
RedisConf: redis.RedisConf{
Host: conf.Addr,
Pass: conf.Password,
Tls: conf.TLS,
Type: conf.Type,
},
Weight: conf.Weight,
})
}
return cacheConf
}
type RedisHook struct {
Prefix string `yaml:"Prefix" json:"prefix,optional"`
}
func (s *RedisHook) DialHook(next redis0.DialHook) redis0.DialHook {
return next
}
func (s *RedisHook) ProcessHook(next redis0.ProcessHook) redis0.ProcessHook {
return next
}
func (s *RedisHook) ProcessPipelineHook(next redis0.ProcessPipelineHook) redis0.ProcessPipelineHook {
return next
}
//func (s *XzRedis) WrapKey(key string) string {
// return s.KeyPrefix + ":" + key
//}
go-db/go-redis/go-redisc/client.go (19.2 KiB)
package goredisc
import (
"context"
gojob "github.com/gif-gif/go.io/go-job"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gogf/gf/container/garray"
"github.com/redis/go-redis/v9"
"github.com/samber/lo"
"time"
)
type GoRedisC struct {
Redis *redis.ClusterClient
Config Config
Ctx context.Context
}
func New(conf Config) (cli *GoRedisC, err error) {
if conf.Type == "" {
conf.Type = "node"
}
cli = &GoRedisC{
Ctx: context.Background(),
Config: conf,
}
//兼容集群模式和单点连接
clusterSlots := func(ctx context.Context) ([]redis.ClusterSlot, error) {
// 自定义集群槽位获取逻辑,通常使用默认值即可
return nil, nil
}
if conf.Type != "node" {
clusterSlots = nil
}
cli.Redis = redis.NewClusterClient(&redis.ClusterOptions{
// 指定集群中的节点地址,至少需要一个有效节点
Addrs: conf.Addrs,
// 连接池配置
PoolSize: lo.If(conf.PoolSize == 0, 10).Else(conf.PoolSize),
MinIdleConns: 5, // 最小空闲连接数
// 认证信息(如果需要)
Password: conf.Password, // 如果有密码,请设置
// 读写超时设置
DialTimeout: lo.If(conf.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.DialTimeout) * time.Second),
ReadTimeout: lo.If(conf.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.ReadTimeout) * time.Second),
WriteTimeout: lo.If(conf.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.WriteTimeout) * time.Second),
// 最大重试次数
MaxRetries: 3,
// 集群刷新间隔
ClusterSlots: clusterSlots,
})
ctx := context.Background()
if err = cli.Redis.Ping(ctx).Err(); err != nil {
golog.WithTag("goredisc").Error(err)
return
}
if conf.AutoPing {
gj, _ := gojob.New()
gj.Start()
gj.SecondX(nil, 5, func() {
if err := cli.Redis.Ping(ctx).Err(); err != nil {
golog.WithTag("goredisc").Warn("redis ping error:", err)
}
})
}
return cli, nil
}
func (s *GoRedisC) WrapKey(key string) string {
if s.Config.Prefix == "" {
return key
}
return s.Config.Prefix + ":" + key
}
func (s *GoRedisC) WrapKeys(keys ...string) []string {
arr := garray.NewStrArrayFromCopy(keys)
return arr.Walk(func(val string) string { return s.WrapKey(val) }).Slice()
}
// BitCount is redis bitcount command implementation.
func (s *GoRedisC) BitCount(key string, start, end int64) *redis.IntCmd {
return s.Redis.BitCount(s.Ctx, s.WrapKey(key), &redis.BitCount{Start: start, End: end})
}
// BitOpAnd is redis bit operation (and) command implementation.
func (s *GoRedisC) BitOpAnd(destKey string, keys ...string) *redis.IntCmd {
return s.Redis.BitOpAnd(s.Ctx, s.WrapKey(destKey), s.WrapKeys(keys...)...)
}
// BitOpNot is redis bit operation (not) command implementation.
func (s *GoRedisC) BitOpNot(destKey, key string) *redis.IntCmd {
return s.Redis.BitOpNot(s.Ctx, s.WrapKey(destKey), s.WrapKey(key))
}
// BitOpOr is redis bit operation (or) command implementation.
func (s *GoRedisC) BitOpOr(destKey string, keys ...string) *redis.IntCmd {
return s.Redis.BitOpOr(s.Ctx, s.WrapKey(destKey), s.WrapKeys(keys...)...)
}
// BitOpXor is redis bit operation (xor) command implementation.
func (s *GoRedisC) BitOpXor(destKey string, keys ...string) *redis.IntCmd {
return s.Redis.BitOpXor(s.Ctx, s.WrapKey(destKey), s.WrapKeys(keys...)...)
}
// BitPos is redis bitpos command implementation.
func (s *GoRedisC) BitPos(key string, bit, start, end int64) *redis.IntCmd {
return s.Redis.BitPos(s.Ctx, s.WrapKey(key), bit, start, end)
}
// Blpop uses passed in redis connection to execute blocking queries.
// Doesn't benefit from pooling redis connections of blocking queries
func (s *GoRedisC) BLPop(timeout time.Duration, key string) *redis.StringSliceCmd {
return s.Redis.BLPop(s.Ctx, timeout, s.WrapKey(key))
}
// Del deletes keys.
func (s *GoRedisC) Del(keys ...string) *redis.IntCmd {
return s.Redis.Del(s.Ctx, s.WrapKeys(keys...)...)
}
// Eval is the implementation of redis eval command.
func (s *GoRedisC) Eval(script string, keys []string, args ...interface{}) *redis.Cmd {
return s.Redis.Eval(s.Ctx, script, s.WrapKeys(keys...), args...)
}
// EvalSha is the implementation of redis evalsha command.
func (s *GoRedisC) EvalSha(sha string, keys []string, args ...interface{}) *redis.Cmd {
return s.Redis.EvalSha(s.Ctx, sha, s.WrapKeys(keys...), args...)
}
// Exists is the implementation of redis exists command.
func (s *GoRedisC) Exists(key string) *redis.IntCmd {
return s.Redis.Exists(s.Ctx, s.WrapKey(key))
}
// Expire is the implementation of redis expire command.
func (s *GoRedisC) Expire(key string, seconds int) *redis.BoolCmd {
return s.Redis.Expire(s.Ctx, s.WrapKey(key), time.Duration(seconds)*time.Second)
}
// Expireat is the implementation of redis expireat command.
func (s *GoRedisC) ExpireAt(key string, time time.Time) *redis.BoolCmd {
return s.Redis.ExpireAt(s.Ctx, s.WrapKey(key), time)
}
// GeoAdd is the implementation of redis geoadd command.
func (s *GoRedisC) GeoAdd(key string, geoLocation ...*redis.GeoLocation) *redis.IntCmd {
return s.Redis.GeoAdd(s.Ctx, s.WrapKey(key), geoLocation...)
}
// GeoDist is the implementation of redis geodist command.
func (s *GoRedisC) GeoDist(key, member1, member2, unit string) *redis.FloatCmd {
return s.Redis.GeoDist(s.Ctx, s.WrapKey(key), member1, member2, unit)
}
// GeoHash is the implementation of redis geohash command.
func (s *GoRedisC) GeoHash(key string, members ...string) *redis.StringSliceCmd {
return s.Redis.GeoHash(s.Ctx, s.WrapKey(key), members...)
}
// GeoRadius is the implementation of redis georadius command.
func (s *GoRedisC) GeoRadius(key string, longitude, latitude float64, query *redis.GeoRadiusQuery) *redis.GeoLocationCmd {
return s.Redis.GeoRadius(s.Ctx, s.WrapKey(key), longitude, latitude, query)
}
// GeoRadiusByMember is the implementation of redis georadiusbymember command.
func (s *GoRedisC) GeoRadiusByMember(key, member string, query *redis.GeoRadiusQuery) *redis.GeoLocationCmd {
return s.Redis.GeoRadiusByMember(s.Ctx, s.WrapKey(key), member, query)
}
// GeoPos is the implementation of redis geopos command.
func (s *GoRedisC) GeoPos(key string, members ...string) *redis.GeoPosCmd {
return s.Redis.GeoPos(s.Ctx, s.WrapKey(key), members...)
}
// Get is the implementation of redis get command.
func (s *GoRedisC) Get(key string) *redis.StringCmd {
return s.Redis.Get(s.Ctx, s.WrapKey(key))
}
// GetBit is the implementation of redis getbit command.
func (s *GoRedisC) GetBit(key string, offset int64) *redis.IntCmd {
return s.Redis.GetBit(s.Ctx, s.WrapKey(key), offset)
}
// Hdel is the implementation of redis hdel command.
func (s *GoRedisC) HDel(key string, fields ...string) *redis.IntCmd {
return s.Redis.HDel(s.Ctx, s.WrapKey(key), fields...)
}
// Hexists is the implementation of redis hexists command.
func (s *GoRedisC) HExists(key, field string) *redis.BoolCmd {
return s.Redis.HExists(s.Ctx, s.WrapKey(key), field)
}
// Hget is the implementation of redis hget command.
func (s *GoRedisC) HGet(key, field string) *redis.StringCmd {
return s.Redis.HGet(s.Ctx, s.WrapKey(key), field)
}
// Hgetall is the implementation of redis hgetall command.
func (s *GoRedisC) HGetAll(key string) *redis.MapStringStringCmd {
return s.Redis.HGetAll(s.Ctx, s.WrapKey(key))
}
// Hincrby is the implementation of redis hincrby command.
func (s *GoRedisC) HIncrBy(key, field string, increment int64) *redis.IntCmd {
return s.Redis.HIncrBy(s.Ctx, s.WrapKey(key), field, increment)
}
// Hkeys is the implementation of redis hkeys command.
func (s *GoRedisC) HKeys(key string) *redis.StringSliceCmd {
return s.Redis.HKeys(s.Ctx, s.WrapKey(key))
}
// Hlen is the implementation of redis hlen command.
func (s *GoRedisC) HLen(key string) *redis.IntCmd {
return s.Redis.HLen(s.Ctx, s.WrapKey(key))
}
// Hmget is the implementation of redis hmget command.
func (s *GoRedisC) HMGet(key string, fields ...string) *redis.SliceCmd {
return s.Redis.HMGet(s.Ctx, s.WrapKey(key), fields...)
}
// Hset is the implementation of redis hset command.
func (s *GoRedisC) HSet(key, field, value string) *redis.IntCmd {
return s.Redis.HSet(s.Ctx, s.WrapKey(key), field, value)
}
// Hsetnx is the implementation of redis hsetnx command.
func (s *GoRedisC) HSetNX(key, field, value string) *redis.BoolCmd {
return s.Redis.HSetNX(s.Ctx, s.WrapKey(key), field, value)
}
// Hmset is the implementation of redis hmset command.
func (s *GoRedisC) HMSet(key string, fieldsAndValues map[string]string) *redis.BoolCmd {
return s.Redis.HMSet(s.Ctx, s.WrapKey(key), fieldsAndValues)
}
// Hscan is the implementation of redis hscan command.
func (s *GoRedisC) HScan(key string, cursor uint64, match string, count int64) *redis.ScanCmd {
return s.Redis.HScan(s.Ctx, s.WrapKey(key), cursor, match, count)
}
// Hvals is the implementation of redis hvals command.
func (s *GoRedisC) HVals(key string) *redis.StringSliceCmd {
return s.Redis.HVals(s.Ctx, s.WrapKey(key))
}
// Incr is the implementation of redis incr command.
func (s *GoRedisC) Incr(key string) *redis.IntCmd {
return s.Redis.Incr(s.Ctx, s.WrapKey(key))
}
// Incrby is the implementation of redis incrby command.
func (s *GoRedisC) IncrBy(key string, increment int64) *redis.IntCmd {
return s.Redis.IncrBy(s.Ctx, s.WrapKey(key), increment)
}
// Keys is the implementation of redis keys command.
func (s *GoRedisC) Keys(pattern string) *redis.StringSliceCmd {
return s.Redis.Keys(s.Ctx, pattern)
}
// Llen is the implementation of redis llen command.
func (s *GoRedisC) LLen(key string) *redis.IntCmd {
return s.Redis.LLen(s.Ctx, s.WrapKey(key))
}
// Lpop is the implementation of redis lpop command.
func (s *GoRedisC) LPop(key string) *redis.StringCmd {
return s.Redis.LPop(s.Ctx, s.WrapKey(key))
}
// Lpush is the implementation of redis lpush command.
func (s *GoRedisC) LPush(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.LPush(s.Ctx, s.WrapKey(key), values...)
}
// Lrange is the implementation of redis lrange command.
func (s *GoRedisC) LRange(key string, start, stop int64) *redis.StringSliceCmd {
return s.Redis.LRange(s.Ctx, s.WrapKey(key), start, stop)
}
// Lrem is the implementation of redis lrem command.
func (s *GoRedisC) LRem(key string, count int64, value string) *redis.IntCmd {
return s.Redis.LRem(s.Ctx, s.WrapKey(key), count, value)
}
// Mget is the implementation of redis mget command.
func (s *GoRedisC) MGet(keys ...string) *redis.SliceCmd {
return s.Redis.MGet(s.Ctx, s.WrapKeys(keys...)...)
}
// Persist is the implementation of redis persist command.
func (s *GoRedisC) Persist(key string) *redis.BoolCmd {
return s.Redis.Persist(s.Ctx, s.WrapKey(key))
}
// Pfadd is the implementation of redis pfadd command.
func (s *GoRedisC) PFAdd(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.PFAdd(s.Ctx, s.WrapKey(key), values...)
}
// Pfcount is the implementation of redis pfcount command.
func (s *GoRedisC) PFCount(key string) *redis.IntCmd {
return s.Redis.PFCount(s.Ctx, s.WrapKey(key))
}
// Pfmerge is the implementation of redis pfmerge command.
func (s *GoRedisC) PFMerge(dest string, keys ...string) *redis.StatusCmd {
return s.Redis.PFMerge(s.Ctx, dest, s.WrapKeys(keys...)...)
}
// Ping is the implementation of redis ping command.
func (s *GoRedisC) Ping() *redis.StatusCmd {
return s.Redis.Ping(s.Ctx)
}
// Pipelined lets fn to execute pipelined commands.
// fn key must call GetKey or GetKeys to add prefix.
func (s *GoRedisC) Pipelined(fn func(redis.Pipeliner) error) ([]redis.Cmder, error) {
return s.Redis.Pipelined(s.Ctx, fn)
}
// Rpop is the implementation of redis rpop command.
func (s *GoRedisC) RPop(key string) *redis.StringCmd {
return s.Redis.RPop(s.Ctx, s.WrapKey(key))
}
// Rpush is the implementation of redis rpush command.
func (s *GoRedisC) RPush(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.RPush(s.Ctx, s.WrapKey(key), values...)
}
// Sadd is the implementation of redis sadd command.
func (s *GoRedisC) SAdd(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.SAdd(s.Ctx, s.WrapKey(key), values...)
}
// Scan is the implementation of redis scan command.
func (s *GoRedisC) Scan(cursor uint64, match string, count int64) *redis.ScanCmd {
return s.Redis.Scan(s.Ctx, cursor, match, count)
}
// SetBit is the implementation of redis setbit command.
func (s *GoRedisC) SetBit(key string, offset int64, value int) *redis.IntCmd {
return s.Redis.SetBit(s.Ctx, s.WrapKey(key), offset, value)
}
// Sscan is the implementation of redis sscan command.
func (s *GoRedisC) SScan(key string, cursor uint64, match string, count int64) *redis.ScanCmd {
return s.Redis.SScan(s.Ctx, s.WrapKey(key), cursor, match, count)
}
// Scard is the implementation of redis scard command.
func (s *GoRedisC) SCard(key string) *redis.IntCmd {
return s.Redis.SCard(s.Ctx, s.WrapKey(key))
}
// ScriptLoad is the implementation of redis script load command.
func (s *GoRedisC) ScriptLoad(script string) *redis.StringCmd {
return s.Redis.ScriptLoad(s.Ctx, script)
}
// Set is the implementation of redis set command.
func (s *GoRedisC) Set(key, value string) *redis.StatusCmd {
return s.Redis.Set(s.Ctx, s.WrapKey(key), value, 0)
}
func (s *GoRedisC) Set1(key, value string, expiration time.Duration) *redis.StatusCmd {
return s.Redis.Set(s.Ctx, s.WrapKey(key), value, expiration)
}
// Setex is the implementation of redis setex command.
func (s *GoRedisC) SetEx(key, value string, expiration time.Duration) *redis.StatusCmd {
return s.Redis.SetEx(s.Ctx, s.WrapKey(key), value, expiration)
}
func (s *GoRedisC) SetNX(key, value string, expiration time.Duration) *redis.BoolCmd {
return s.Redis.SetNX(s.Ctx, s.WrapKey(key), value, expiration)
}
// Sismember is the implementation of redis sismember command.
func (s *GoRedisC) SIsMember(key string, value interface{}) *redis.BoolCmd {
return s.Redis.SIsMember(s.Ctx, s.WrapKey(key), value)
}
// Smembers is the implementation of redis smembers command.
func (s *GoRedisC) SMembers(key string) *redis.StringSliceCmd {
return s.Redis.SMembers(s.Ctx, s.WrapKey(key))
}
// Spop is the implementation of redis spop command.
func (s *GoRedisC) SPop(key string) *redis.StringCmd {
return s.Redis.SPop(s.Ctx, s.WrapKey(key))
}
// Srandmember is the implementation of redis srandmember command.
func (s *GoRedisC) SRandMember(key string) *redis.StringCmd {
return s.Redis.SRandMember(s.Ctx, s.WrapKey(key))
}
// Srem is the implementation of redis srem command.
func (s *GoRedisC) SRem(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.SRem(s.Ctx, s.WrapKey(key), values...)
}
// Sunion is the implementation of redis sunion command.
func (s *GoRedisC) SUnion(keys ...string) *redis.StringSliceCmd {
return s.Redis.SUnion(s.Ctx, s.WrapKeys(keys...)...)
}
// Sunionstore is the implementation of redis sunionstore command.
func (s *GoRedisC) SUnionStore(destination string, keys ...string) *redis.IntCmd {
return s.Redis.SUnionStore(s.Ctx, destination, s.WrapKeys(keys...)...)
}
// Sdiff is the implementation of redis sdiff command.
func (s *GoRedisC) SDiff(keys ...string) *redis.StringSliceCmd {
return s.Redis.SDiff(s.Ctx, s.WrapKeys(keys...)...)
}
// Sdiffstore is the implementation of redis sdiffstore command.
func (s *GoRedisC) SDiffStore(destination string, keys ...string) *redis.IntCmd {
return s.Redis.SDiffStore(s.Ctx, destination, s.WrapKeys(keys...)...)
}
// Sinter is the implementation of redis sinter command.
func (s *GoRedisC) SInter(keys ...string) *redis.StringSliceCmd {
return s.Redis.SInter(s.Ctx, s.WrapKeys(keys...)...)
}
// Sinterstore is the implementation of redis sinterstore command.
func (s *GoRedisC) SInterStore(destination string, keys ...string) *redis.IntCmd {
return s.Redis.SInterStore(s.Ctx, destination, s.WrapKeys(keys...)...)
}
// Ttl is the implementation of redis ttl command.
func (s *GoRedisC) TTL(key string) *redis.DurationCmd {
return s.Redis.TTL(s.Ctx, s.WrapKey(key))
}
// Zadd is the implementation of redis zadd command.
func (s *GoRedisC) ZAdd(key string, value ...redis.Z) *redis.IntCmd {
return s.Redis.ZAdd(s.Ctx, s.WrapKey(key), value...)
}
// Zcard is the implementation of redis zcard command.
func (s *GoRedisC) ZCard(key string) *redis.IntCmd {
return s.Redis.ZCard(s.Ctx, s.WrapKey(key))
}
// Zcount is the implementation of redis zcount command.
func (s *GoRedisC) ZCount(key string, max, min string) *redis.IntCmd {
return s.Redis.ZCount(s.Ctx, s.WrapKey(key), min, max)
}
// Zincrby is the implementation of redis zincrby command.
func (s *GoRedisC) ZIncrBy(key string, increment float64, field string) *redis.FloatCmd {
return s.Redis.ZIncrBy(s.Ctx, s.WrapKey(key), increment, field)
}
// Zscore is the implementation of redis zscore command.
func (s *GoRedisC) ZScore(key, value string) *redis.FloatCmd {
return s.Redis.ZScore(s.Ctx, s.WrapKey(key), value)
}
// Zrank is the implementation of redis zrank command.
func (s *GoRedisC) ZRank(key, field string) *redis.IntCmd {
return s.Redis.ZRank(s.Ctx, s.WrapKey(key), field)
}
// Zrem is the implementation of redis zrem command.
func (s *GoRedisC) Zrem(key string, values ...interface{}) *redis.IntCmd {
return s.Redis.ZRem(s.Ctx, s.WrapKey(key), values...)
}
// Zremrangebyscore is the implementation of redis zremrangebyscore command.
func (s *GoRedisC) ZRemRangeByScore(key string, max, min string) *redis.IntCmd {
return s.Redis.ZRemRangeByScore(s.Ctx, s.WrapKey(key), min, max)
}
// Zremrangebyrank is the implementation of redis zremrangebyrank command.
func (s *GoRedisC) Zremrangebyrank(key string, start, stop int64) *redis.IntCmd {
return s.Redis.ZRemRangeByRank(s.Ctx, s.WrapKey(key), start, stop)
}
// Zrange is the implementation of redis zrange command.
func (s *GoRedisC) Zrange(key string, start, stop int64) *redis.StringSliceCmd {
return s.Redis.ZRange(s.Ctx, s.WrapKey(key), start, stop)
}
// ZrangeWithScores is the implementation of redis zrange command with scores.
func (s *GoRedisC) ZRangeWithScores(key string, start, stop int64) *redis.ZSliceCmd {
return s.Redis.ZRangeWithScores(s.Ctx, s.WrapKey(key), start, stop)
}
// ZRevRangeWithScores is the implementation of redis zrevrange command with scores.
func (s *GoRedisC) ZRevRangeWithScores(key string, start, stop int64) *redis.ZSliceCmd {
return s.Redis.ZRevRangeWithScores(s.Ctx, s.WrapKey(key), start, stop)
}
// ZrangebyscoreWithScores is the implementation of redis zrangebyscore command with scores.
func (s *GoRedisC) ZRangeByScoreWithScores(key string, opt *redis.ZRangeBy) *redis.ZSliceCmd {
return s.Redis.ZRangeByScoreWithScores(s.Ctx, s.WrapKey(key), opt)
}
// Zrevrange is the implementation of redis zrevrange command.
func (s *GoRedisC) Zrevrange(key string, start, stop int64) *redis.StringSliceCmd {
return s.Redis.ZRevRange(s.Ctx, s.WrapKey(key), start, stop)
}
// ZrevrangebyscoreWithScores is the implementation of redis zrevrangebyscore command with scores.
func (s *GoRedisC) ZRevRangeByScoreWithScores(key string, opt *redis.ZRangeBy) *redis.ZSliceCmd {
return s.Redis.ZRevRangeByScoreWithScores(s.Ctx, s.WrapKey(key), opt)
}
// Zrevrank is the implementation of redis zrevrank command.
func (s *GoRedisC) ZRevRank(key, field string) *redis.IntCmd {
return s.Redis.ZRevRank(s.Ctx, s.WrapKey(key), field)
}
// Zunionstore is the implementation of redis zunionstore command.
func (s *GoRedisC) ZUnionStore(dest string, store *redis.ZStore) *redis.IntCmd {
return s.Redis.ZUnionStore(s.Ctx, dest, store)
}
func (s *GoRedisC) Close() error {
return s.Redis.Close()
}
go-db/go-redis/go-redisc/config.go (1.9 KiB)
package goredisc
import (
"github.com/zeromicro/go-zero/core/stores/cache"
"github.com/zeromicro/go-zero/core/stores/redis"
"strings"
"time"
)
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
//NamePrefix string `yaml:"NamePrefix" json:"namePrefix,optional"`
Addrs []string `yaml:"Addrs" json:"addrs,optional"`
Password string `yaml:"Password" json:"password,optional"`
DB int `yaml:"DB" json:"db,optional"`
Prefix string `yaml:"Prefix" json:"prefix,optional"`
AutoPing bool `yaml:"AutoPing" json:"autoPing,optional"`
TLS bool `yaml:"TLS" json:"tls,optional"`
DialTimeout int `yaml:"DialTimeout" json:"dialTimeout,optional"`
ReadTimeout int `yaml:"ReadTimeout" json:"readTimeout,optional"`
WriteTimeout int `yaml:"WriteTimeout" json:"writeTimeout,optional"`
MaxRedirects int `yaml:"MaxRedirects" json:"maxRedirects,optional"`
PoolSize int `yaml:"PoolSize" json:"poolSize,optional"`
Type string `yaml:"Type" json:",default=node,options=node|cluster"`
PingTimeout int64 `yaml:"PingTimeout" json:"pingTimeout,optional"`
Weight int `yaml:"Weight" json:",default=100"` //for gozero TODO: 这里需要有多个节点的配置
}
type ClusterConf Config
// 兼容go-zore
func (c ClusterConf) GetCacheConf() cache.CacheConf {
cacheConf := make([]cache.NodeConf, 0, len(c.Addrs))
addrs := make([]string, 0, len(c.Addrs))
for _, addr := range c.Addrs {
addrs = append(addrs, addr)
}
if c.PingTimeout == 0 {
c.PingTimeout = 10
}
cacheConf = append(cacheConf, cache.NodeConf{
RedisConf: redis.RedisConf{
Host: strings.Join(addrs, ","),
Pass: c.Password,
Tls: c.TLS,
Type: c.Type,
PingTimeout: time.Duration(c.PingTimeout) * time.Second,
},
Weight: c.Weight,
})
return cacheConf
}
func (c ClusterConf) GetConfig() Config {
return Config(c)
}
go-db/go-redis/go-redisc/redis.go (1.2 KiB)
package goredisc
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoRedisC{}
// 可以一次初始化多个Redis实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("goredisc client [" + name + "] already exists")
}
__clients[name], err = New(conf)
if err != nil {
return err
}
}
return
}
func GetClient(names ...string) *GoRedisC {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
r := GetClient(name)
if r != nil {
r.Close()
}
delete(__clients, name)
}
}
}
func Default() *GoRedisC {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("goredisc").Error("no default redis client")
return nil
}
go-db/go-redis/go-redisc/utils.go (7.7 KiB)
package goredisc
import (
"context"
"fmt"
"sort"
"sync"
"time"
"github.com/redis/go-redis/v9"
)
type LatencyStats struct {
Samples map[string][]time.Duration `json:"-"` // 原始样本数据不序列化
Total int64 `json:"total"`
Avg time.Duration `json:"avg"`
mu sync.RWMutex `json:"-"` // 互斥锁不序列化
}
// CommandStats 单个命令的统计信息
type CommandStats struct {
Command string `json:"command"`
Count int `json:"count"`
Average time.Duration `json:"average"`
Min time.Duration `json:"min"`
Max time.Duration `json:"max"`
P50 time.Duration `json:"p50"`
P90 time.Duration `json:"p90"`
P95 time.Duration `json:"p95"`
P99 time.Duration `json:"p99"`
P999 time.Duration `json:"p999"`
}
// LatencyStatsJSON JSON 序列化结构体
type LatencyStatsJSON struct {
TotalSamples int64 `json:"total_samples"`
OverallAvg time.Duration `json:"overall_avg"`
Commands []CommandStats `json:"commands"`
Timestamp time.Time `json:"timestamp"`
}
// LatencyStatsJSONString JSON 序列化结构体(字符串格式的时间)
type LatencyStatsJSONString struct {
TotalSamples int64 `json:"total_samples"`
OverallAvg string `json:"overall_avg"`
Commands []CommandStatsString `json:"commands"`
Timestamp string `json:"timestamp"`
}
// CommandStatsString 单个命令的统计信息(字符串格式的时间)
type CommandStatsString struct {
Command string `json:"command"`
Count int `json:"count"`
Average string `json:"average"`
Min string `json:"min"`
Max string `json:"max"`
P50 string `json:"p50"`
P90 string `json:"p90"`
P95 string `json:"p95"`
P99 string `json:"p99"`
P999 string `json:"p999"`
}
// NewLatencyStats 创建新的延迟统计实例
func NewLatencyStats() *LatencyStats {
return &LatencyStats{
Samples: make(map[string][]time.Duration),
}
}
// AddSample 添加延迟样本
func (ls *LatencyStats) AddSample(command string, latency time.Duration) {
ls.mu.Lock()
defer ls.mu.Unlock()
if ls.Samples[command] == nil {
ls.Samples[command] = make([]time.Duration, 0)
}
ls.Samples[command] = append(ls.Samples[command], latency)
ls.Total++
// 重新计算平均值
ls.calculateAverage()
}
// calculateAverage 计算总体平均延迟
func (ls *LatencyStats) calculateAverage() {
var totalDuration time.Duration
var count int64
for _, samples := range ls.Samples {
for _, sample := range samples {
totalDuration += sample
count++
}
}
if count > 0 {
ls.Avg = totalDuration / time.Duration(count)
}
}
// GetCommandPercentile 获取指定命令的百分位延迟
func (ls *LatencyStats) GetCommandPercentile(command string, percentile float64) time.Duration {
ls.mu.RLock()
defer ls.mu.RUnlock()
samples := ls.Samples[command]
if len(samples) == 0 {
return 0
}
sorted := make([]time.Duration, len(samples))
copy(sorted, samples)
sort.Slice(sorted, func(i, j int) bool {
return sorted[i] < sorted[j]
})
index := int(float64(len(sorted)) * percentile / 100.0)
if index >= len(sorted) {
index = len(sorted) - 1
}
return sorted[index]
}
// GetCommandAverage 获取指定命令的平均延迟
func (ls *LatencyStats) GetCommandAverage(command string) time.Duration {
ls.mu.RLock()
defer ls.mu.RUnlock()
samples := ls.Samples[command]
if len(samples) == 0 {
return 0
}
var total time.Duration
for _, sample := range samples {
total += sample
}
return total / time.Duration(len(samples))
}
// getCommandMinMax 获取指定命令的最小值和最大值
func (ls *LatencyStats) getCommandMinMax(command string) (time.Duration, time.Duration) {
ls.mu.RLock()
defer ls.mu.RUnlock()
samples := ls.Samples[command]
if len(samples) == 0 {
return 0, 0
}
min, max := samples[0], samples[0]
for _, sample := range samples {
if sample < min {
min = sample
}
if sample > max {
max = sample
}
}
return min, max
}
// GetCommandCount 获取指定命令的样本数量
func (ls *LatencyStats) GetCommandCount(command string) int {
ls.mu.RLock()
defer ls.mu.RUnlock()
return len(ls.Samples[command])
}
// GetAllCommands 获取所有测试的命令列表
func (ls *LatencyStats) GetAllCommands() []string {
ls.mu.RLock()
defer ls.mu.RUnlock()
commands := make([]string, 0, len(ls.Samples))
for cmd := range ls.Samples {
commands = append(commands, cmd)
}
return commands
}
// ToJSON 将延迟统计转换为 JSON 字节数组(time.Duration 格式)
func (ls *LatencyStats) ToJSON() (LatencyStatsJSON, error) {
ls.mu.RLock()
defer ls.mu.RUnlock()
jsonData := LatencyStatsJSON{
TotalSamples: ls.Total,
OverallAvg: ls.Avg,
Commands: make([]CommandStats, 0),
Timestamp: time.Now(),
}
for cmd := range ls.Samples {
if len(ls.Samples[cmd]) == 0 {
continue
}
min, max := ls.getCommandMinMax(cmd)
cmdStats := CommandStats{
Command: cmd,
Count: ls.GetCommandCount(cmd),
Average: ls.GetCommandAverage(cmd),
Min: min,
Max: max,
P50: ls.GetCommandPercentile(cmd, 50),
P90: ls.GetCommandPercentile(cmd, 90),
P95: ls.GetCommandPercentile(cmd, 95),
P99: ls.GetCommandPercentile(cmd, 99),
P999: ls.GetCommandPercentile(cmd, 99.9),
}
jsonData.Commands = append(jsonData.Commands, cmdStats)
}
return jsonData, nil
}
// ToJSONString 将延迟统计转换为 JSON 字节数组(字符串格式的时间,便于阅读)
func (ls *LatencyStats) ToJSONString() (LatencyStatsJSONString, error) {
ls.mu.RLock()
defer ls.mu.RUnlock()
jsonData := LatencyStatsJSONString{
TotalSamples: ls.Total,
OverallAvg: ls.Avg.String(),
Commands: make([]CommandStatsString, 0),
Timestamp: time.Now().Format("2006-01-02 15:04:05"),
}
for cmd := range ls.Samples {
if len(ls.Samples[cmd]) == 0 {
continue
}
min, max := ls.getCommandMinMax(cmd)
cmdStats := CommandStatsString{
Command: cmd,
Count: ls.GetCommandCount(cmd),
Average: ls.GetCommandAverage(cmd).String(),
Min: min.String(),
Max: max.String(),
P50: ls.GetCommandPercentile(cmd, 50).String(),
P90: ls.GetCommandPercentile(cmd, 90).String(),
P95: ls.GetCommandPercentile(cmd, 95).String(),
P99: ls.GetCommandPercentile(cmd, 99).String(),
P999: ls.GetCommandPercentile(cmd, 99.9).String(),
}
jsonData.Commands = append(jsonData.Commands, cmdStats)
}
return jsonData, nil
}
// benchmarkRedisLatency 使用新的结构进行延迟基准测试
func BenchmarkRedisLatency(rdb *redis.ClusterClient, duration time.Duration) *LatencyStats {
ctx := context.Background()
stats := NewLatencyStats()
fmt.Printf("Running latency benchmark for %v...\n", duration)
start := time.Now()
// 定义要测试的命令
commands := map[string]func() error{
"PING": func() error {
return rdb.Ping(ctx).Err()
},
"SET": func() error {
return rdb.Set(ctx, fmt.Sprintf("test_key_%d", time.Now().UnixNano()), "test_value", 0).Err()
},
"GET": func() error {
return rdb.Get(ctx, "test_key").Err()
},
"INCR": func() error {
return rdb.Incr(ctx, "test_counter").Err()
},
"LPUSH": func() error {
return rdb.LPush(ctx, "test_list", "test_value").Err()
},
}
commandNames := make([]string, 0, len(commands))
for cmd := range commands {
commandNames = append(commandNames, cmd)
}
cmdIndex := 0
for time.Since(start) < duration {
// 轮询执行不同命令
cmdName := commandNames[cmdIndex%len(commandNames)]
cmdFunc := commands[cmdName]
cmdStart := time.Now()
err := cmdFunc()
latency := time.Since(cmdStart)
if err == nil {
stats.AddSample(cmdName, latency)
}
cmdIndex++
// 控制请求频率
time.Sleep(10 * time.Millisecond)
}
return stats
}
go-db/go-redis/redis.go (1.2 KiB)
package goredis
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoRedis{}
// 可以一次初始化多个Redis实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("goredis client [" + name + "] already exists")
}
__clients[name], err = New(conf)
if err != nil {
return err
}
}
return
}
func GetClient(names ...string) *GoRedis {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
r := GetClient(name)
if r != nil {
r.Close()
}
delete(__clients, name)
}
}
}
func Default() *GoRedis {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("goredis").Error("no default redis client")
return nil
}
go-db/go-redis/test/main.go (605 B)
package main
import (
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
golog "github.com/gif-gif/go.io/go-log"
)
func main() {
config := goredisc.Config{
Name: "goredis",
Addrs: []string{"127.0.0.1:6379"},
Password: "",
DB: 0,
Prefix: "goredis",
AutoPing: true,
}
err := goredisc.Init(config)
if err != nil {
golog.WithTag("goredis").Error(err)
}
cmd := goredisc.Default().Set("goredis", "goredis")
if cmd.Err() != nil {
golog.WithTag("goredis").Error(cmd.Err())
}
v := goredisc.Default().Get("goredis").Val()
golog.WithTag("goredis").InfoF(v)
}
go-db/gogorm/ReadMe.md (3.0 KiB)
go-db 数据操作
- https://www.topgoer.com/%E6%95%B0%E6%8D%AE%E5%BA%93%E6%93%8D%E4%BD%9C/
基于 https://github.com/go-gorm/gorm 的封装,更多用法到官方
目前已封装
- mysql
- sqlite3
- clickhouse
- postgresql
- sqlserver
- tidb
func testSqlite3() {
err := gogorm.Init(gogorm.Config{
DataSource: "./test.db",
DBType: gogorm.DATABASE_SQLITE,
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
// Create
insertProduct := &Product{Code: "D42", Price: 100}
db.Insert(insertProduct)
fmt.Println(insertProduct.ID)
// Read
var product Product
tx := db.First(&product, 1) // find product with integer primary key
if tx.Error != nil {
fmt.Println("not found first ", tx.Error.Error())
}
db.First(&product, "code = ?", "D42")
// Delete - delete product
db.Delete(&product, 1)
err = goutils.RemoveFile("./test.db")
if err != nil {
golog.WithTag("godb").Error(err.Error())
}
}
func mysqlTest() {
err := gogorm.Init(gogorm.Config{
DataSource: "root:223238@tcp(127.0.0.1:33060)/gromdb?charset=utf8mb4&parseTime=True&loc=Localb",
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
// Create
insertProduct := &Product{Code: "D42", Price: 100}
db.Insert(insertProduct)
fmt.Println(insertProduct.ID)
// Read
var product Product
tx := db.First(&product, 1) // find product with integer primary key
if tx.Error != nil {
fmt.Println("not found first ", tx.Error.Error())
}
db.First(&product, "code = ?", "D42")
// Delete - delete product
db.Delete(&product, 1)
}
func testClickhouse() {
dsn := "tcp://localhost:9000?database=gorm&username=gorm&password=gorm&read_timeout=10&write_timeout=20"
err := gogorm.Init(gogorm.Config{
DataSource: dsn,
DBType: gogorm.DATABASE_CLICKHOUSE,
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.Set("gorm:table_options", "ENGINE=Distributed(cluster, default, hits)").AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
// Set table options
// Create
insertProduct := &Product{Code: "D42", Price: 100}
db.Insert(insertProduct)
fmt.Println(insertProduct.ID)
// Read
var product Product
tx := db.First(&product, 1) // find product with integer primary key
if tx.Error != nil {
fmt.Println("not found first ", tx.Error.Error())
}
db.First(&product, "code = ?", "D42")
// Delete - delete product
db.Delete(&product, 1)
}
mongodb 不是关系性数据库 暂时不支持
go-db/gogorm/client.go (1.5 KiB)
package gogorm
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
"gorm.io/gorm"
)
var _clients = map[string]*GoGorm{}
func GetNames() []string {
names := make([]string, 0, len(_clients))
for name := range _clients {
names = append(names, name)
}
return names
}
func InitWithGormConfig(gormConfig gorm.Config, configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if _clients[name] != nil {
return errors.New("gogorm client already exists")
}
_clients[name], err = New(&conf, gormConfig)
if err != nil {
return
}
}
return
}
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if _clients[name] != nil {
return errors.New("gogorm client already exists")
}
_clients[name], err = New(&conf)
if err != nil {
return
}
}
return
}
func GetClient(names ...string) *GoGorm {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := _clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(_clients, name)
}
}
}
func Default() *GoGorm {
if cli, ok := _clients["default"]; ok {
return cli
}
if l := len(_clients); l == 1 {
for _, cli := range _clients {
return cli
}
}
golog.WithTag("gogorm").Error("no default gogorm client")
return nil
}
go-db/gogorm/clientex.go (1.7 KiB)
package gogorm
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
"gorm.io/gorm"
)
var _clientsForDbType = map[string]*GoGorm{}
func InitByDbTypeWithGormConfig(gormConfig gorm.Config, configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
key := conf.DBType + ":" + name
if _clientsForDbType[key] != nil {
return errors.New("gogorm clientForDbType already exists")
}
_clientsForDbType[key], err = New(&conf, gormConfig)
if err != nil {
return
}
}
return
}
func InitByDbType(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
key := conf.DBType + ":" + name
if _clientsForDbType[key] != nil {
return errors.New("gogorm clientForDbType already exists")
}
_clientsForDbType[key], err = New(&conf)
if err != nil {
return
}
}
return
}
func GetClientByDbType(dbType string, names ...string) *GoGorm {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
key := dbType + ":" + name
if cli, ok := _clientsForDbType[key]; ok {
return cli
}
return nil
}
func DelClientDbType(dbType string, names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
key := dbType + ":" + name
delete(_clientsForDbType, key)
}
}
}
func DefaultForDbType(dbType string) *GoGorm {
if cli, ok := _clientsForDbType[dbType+":default"]; ok {
return cli
}
if l := len(_clientsForDbType); l == 1 {
for _, cli := range _clientsForDbType {
return cli
}
}
golog.WithTag("gogorm").Error("no default gogorm clientForDbType")
return nil
}
go-db/gogorm/config.go (837 B)
package gogorm
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
DBType string `yaml:"DBType" json:"dbType,optional"` // default mysql
DataSource string `yaml:"DataSource" json:"dataSource,optional"`
MaxIdleCount int `yaml:"MaxIdleCount" json:"maxIdleCount,optional"` // zero means defaultMaxIdleConns; negative means 0
MaxOpen int `yaml:"MaxOpen" json:"maxOpen,optional"` // <= 0 means unlimited
MaxLifetime int `yaml:"MaxLifetime" json:"maxLifetime,optional"` // maximum amount of time a connection may be reused minutes
}
const (
DATABASE_MYSQL = "mysql"
DATABASE_POSTGRESQL = "postgres"
DATABASE_SQLITE = "sqlite"
DATABASE_SQLSERVER = "sqlserver"
DATABASE_CLICKHOUSE = "clickhouse"
DATABASE_TIDB = "tidb"
DATABASE_STARROCKS = "starrocks"
)
go-db/gogorm/go_gorm.go (3.2 KiB)
package gogorm
import (
"github.com/gif-gif/go.io/goio"
"gorm.io/driver/clickhouse"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/driver/sqlserver"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"gorm.io/gorm/schema"
"time"
)
type GoGorm struct {
DB *gorm.DB
Config *Config
GormConfig *gorm.Config
}
func New(config *Config, gormConfig ...gorm.Config) (*GoGorm, error) {
m := &GoGorm{
Config: config,
}
dlt, err := createDialector(config)
if err != nil {
return nil, err
}
if len(gormConfig) > 0 {
m.GormConfig = &gormConfig[0]
}
if m.GormConfig == nil {
m.GormConfig = createDefaultConfig(config)
}
db, err := gorm.Open(dlt, m.GormConfig)
if err != nil {
return nil, err
}
sqlDB, err := db.DB()
if err != nil {
return nil, err
}
if sqlDB != nil {
if config.MaxIdleCount == 0 {
config.MaxIdleCount = 10
}
// SetMaxIdleConns sets the maximum number of connections in the idle connection pool.
sqlDB.SetMaxIdleConns(config.MaxIdleCount)
if config.MaxOpen == 0 {
config.MaxOpen = 100
}
// SetMaxOpenConns sets the maximum number of open connections to the database.
sqlDB.SetMaxOpenConns(config.MaxOpen)
// SetConnMaxLifetime sets the maximum amount of time a connection may be reused.
if config.MaxLifetime == 0 {
sqlDB.SetConnMaxLifetime(time.Hour)
} else {
sqlDB.SetConnMaxLifetime(time.Duration(config.MaxLifetime) * time.Minute)
}
}
m.DB = db
return m, nil
}
func createDialector(config *Config) (gorm.Dialector, error) {
switch config.DBType {
case DATABASE_MYSQL, DATABASE_TIDB:
return mysql.Open(config.DataSource), nil
case DATABASE_STARROCKS:
return mysql.Open(config.DataSource), nil
case DATABASE_POSTGRESQL:
return postgres.New(postgres.Config{
DSN: config.DataSource,
PreferSimpleProtocol: false, // enable implicit prepared statement usage
}), nil
case DATABASE_SQLITE:
return sqlite.Open(config.DataSource), nil
case DATABASE_SQLSERVER:
return sqlserver.Open(config.DataSource), nil
case DATABASE_CLICKHOUSE:
return clickhouse.Open(config.DataSource), nil
default:
return mysql.Open(config.DataSource), nil
}
}
func createDefaultConfig(config *Config) *gorm.Config {
logLevel := logger.Info
if goio.IsPro() {
logLevel = logger.Silent
}
switch config.DBType {
case DATABASE_MYSQL, DATABASE_TIDB:
return &gorm.Config{Logger: logger.Default.LogMode(logLevel)}
case DATABASE_STARROCKS:
return &gorm.Config{
// 使用较详细的日志级别,生产环境可调整
Logger: logger.Default.LogMode(logLevel),
// 关闭自动复数化命名,StarRocks对此支持有限
NamingStrategy: schema.NamingStrategy{
SingularTable: true,
},
// 禁用外键约束,StarRocks不支持外键
DisableForeignKeyConstraintWhenMigrating: true,
}
case DATABASE_POSTGRESQL:
return &gorm.Config{Logger: logger.Default.LogMode(logLevel)}
case DATABASE_SQLITE:
return &gorm.Config{Logger: logger.Default.LogMode(logLevel)}
case DATABASE_SQLSERVER:
return &gorm.Config{Logger: logger.Default.LogMode(logLevel)}
case DATABASE_CLICKHOUSE:
return &gorm.Config{Logger: logger.Default.LogMode(logLevel)}
default:
return &gorm.Config{Logger: logger.Default.LogMode(logLevel)}
}
}
go-db/pageex.go (2.4 KiB)
package godb
//简单的分布组件
import (
"github.com/gogf/gf/util/gconv"
"strconv"
)
type OrderItem struct {
Column string `json:"column"`
Asc bool `json:"asc"`
IsFunc bool `json:"isFunc"` //是否是函数, 默认是false , 如果是true, 则会移除column中的反引号
}
type Page struct {
PageNo int64 `json:"page_no,optional"`
PageSize int64 `json:"page_size,optional"`
StartTime int64 `json:"start_time,optional"`
EndTime int64 `json:"end_time,optional"`
SortBy []*OrderItem `json:"sort_by,optional"`
GroupBy []string `json:"group_by,optional"`
IgnoreTotal bool `json:"ignore_total,optional"`
IgnoreList bool `json:"need_list,optional"`
OnlyTotal bool `json:"only_total,optional"`
Ids []int64 `json:"ids,optional"`
//ExcludeIds []int64 `json:"excludeIds,optional"`
States []int64 `json:"states,optional"`
Statuses []int64 `json:"statutes,optional"`
}
//TODO: 待优化
//type CommonCondition struct {
// ExcludeIds []int64 `json:"exclude_ids,optional"`
//}
func (p *Page) OrderBy() string {
size := len(p.SortBy)
if size == 0 {
return "order by id desc"
}
order := "order by "
for i, v := range p.SortBy {
order = order + v.Column + " "
if !v.Asc {
order = order + " desc "
}
if size-1 == i {
} else {
order = order + ","
}
}
return order
}
func (p *Page) OrderByExt() string {
size := len(p.SortBy)
if size == 0 {
return ""
}
order := ""
for i, v := range p.SortBy {
order = order + v.Column + " "
if !v.Asc {
order = order + " desc "
}
if size-1 == i {
} else {
order = order + ","
}
}
return order
}
func (p *Page) PageLimit() string {
if p.PageNo == 0 {
p.PageNo = 1
}
if p.PageSize == 0 {
p.PageSize = 20
}
return "limit " + gconv.String((p.PageNo-1)*p.PageSize) + "," + gconv.String(p.PageSize)
}
func (p *Page) GroupByStr() string {
size := len(p.GroupBy)
if size == 0 {
return ""
}
order := "group by "
for i, v := range p.GroupBy {
order = order + v + " "
if size-1 == i {
} else {
order = order + ","
}
}
return order
}
func (p *Page) PageTimeRange(filed string) string {
where := " "
if p.StartTime > 0 {
where += " and " + filed + " >= " + strconv.Itoa(int(p.StartTime))
}
if p.EndTime > 0 {
where += " and " + filed + " < " + strconv.Itoa(int(p.EndTime))
}
return where
}
go-db/readme.md (29 B)
Golang 数据库操作集
go-db/select.go (3.3 KiB)
package godb
import (
"github.com/gogf/gf/util/gconv"
"strings"
)
type SelectController[T int64 | string] struct {
Values []T `json:"values,optional"`
Exclude bool `json:"exclude,optional"`
}
func (c *SelectController[T]) ClickHouseWhere(column string) (string, []T) {
if len(c.Values) == 0 {
return "", nil
}
var whereString string
if c.Exclude {
whereString = " not in ? "
} else {
whereString = " in ? "
}
return " " + column + " " + whereString, c.Values
}
func (c *SelectController[T]) MysqlWhere(column string) (string, []any) {
if len(c.Values) == 0 {
return "", nil
}
var whereString string
if c.Exclude {
whereString = column + " not in "
} else {
whereString = column + " in "
}
conditions, params := GenerateSliceIn[T](c.Values)
return whereString + conditions, params
}
func WhereIntArray[T int | int64 | int32](items []int64) string {
if len(items) == 0 {
return ""
}
builder := strings.Builder{}
builder.WriteString(" (")
for i := 0; i < len(items)-1; i++ {
builder.WriteString(gconv.String(items[i]))
builder.WriteString(",")
}
builder.WriteString(gconv.String(items[len(items)-1]))
builder.WriteString(") ")
return builder.String()
}
func WhereStringArray(items []string) string {
if len(items) == 0 {
return ""
}
builder := strings.Builder{}
builder.WriteString(" (")
for i := 0; i < len(items)-1; i++ {
builder.WriteString("'" + items[i] + "',")
}
builder.WriteString("'" + gconv.String(items[len(items)-1]))
builder.WriteString("') ")
return builder.String()
}
func GenerateSliceIn[T any](srcItems []T) (string, []any) {
if len(srcItems) == 0 {
return "", nil
}
targetItems := make([]any, 0, len(srcItems))
builder := strings.Builder{}
builder.WriteString(" ( ")
for _, item := range srcItems {
builder.WriteString("?,")
targetItems = append(targetItems, item)
}
targetString := builder.String()
targetString = targetString[:len(targetString)-1]
return targetString + " ) ", targetItems
}
func GenerateSliceInEx[T any](fieldName string, srcItems []T) (string, []any) {
if len(srcItems) == 0 {
return "", nil
}
targetItems := make([]any, 0, len(srcItems))
builder := strings.Builder{}
builder.WriteString(" ( ")
for _, item := range srcItems {
builder.WriteString("?,")
targetItems = append(targetItems, item)
}
targetString := builder.String()
targetString = targetString[:len(targetString)-1]
return targetString + " ) ", targetItems
}
type NumberRangeController[T int64 | int | float64] struct {
Min *T `json:"min,optional"`
Max *T `json:"max,optional"`
}
func (c *NumberRangeController[T]) Where(column string) (string, []any) {
if c.Min == nil && c.Max == nil {
return "", []any{}
}
var whereString string
args := make([]any, 0, 2)
if c.Min != nil {
whereString = " " + column + " >= ? "
args = append(args, *c.Min)
}
if c.Max != nil {
if len(whereString) > 0 {
whereString += " and " + column + " < ? "
} else {
whereString = " " + column + " < ? "
}
args = append(args, *c.Max)
}
return whereString, args
}
// 返回没有order by 前缀
func (p *Page) ClickHouseOrderByExt() string {
size := len(p.SortBy)
if size == 0 {
return ""
}
order := ""
for i, v := range p.SortBy {
order = order + v.Column + " "
if !v.Asc {
order = order + " desc "
}
if size-1 == i {
} else {
order = order + ","
}
}
return order
}
go-db/test/main.go (6.6 KiB)
package main
import (
"fmt"
"github.com/gif-gif/go.io/go-db/gogorm"
gofile "github.com/gif-gif/go.io/go-file"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"github.com/gogf/gf/util/gconv"
"gorm.io/gorm"
)
type Product struct {
gorm.Model
Code string
Price uint
}
func main() {
goio.Init(goio.DEVELOPMENT)
//testSqlite3()
//mysqlTest()
//testTransaction()
//postgreSqlTest()
//testHasMany()
tarTest()
}
func tarTest() {
err := gogorm.Init(gogorm.Config{
Name: gogorm.DATABASE_MYSQL,
DBType: gogorm.DATABASE_MYSQL,
DataSource: "root:223238@tcp(127.0.0.1:33060)/gromdb?charset=utf8mb4&parseTime=True&loc=Local",
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
}
func testSqlite3() {
err := gogorm.Init(gogorm.Config{
DataSource: "./test.db",
DBType: gogorm.DATABASE_SQLITE,
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
// Create
insertProduct := &Product{Code: "D42", Price: 100}
db.Create(insertProduct)
fmt.Println(insertProduct.ID)
// Read
var product Product
tx := db.First(&product, 1) // find product with integer primary key
if tx.Error != nil {
fmt.Println("not found first ", tx.Error.Error())
}
db.First(&product, "code = ?", "D42")
// Delete - delete product
db.Delete(&product, 1)
err = gofile.RemoveFile("./test.db")
if err != nil {
golog.WithTag("godb").Error(err.Error())
}
}
func mysqlTest() {
err := gogorm.Init(gogorm.Config{
DataSource: "root:223238@tcp(127.0.0.1:33060)/gromdb?charset=utf8mb4&parseTime=True&loc=Local",
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
// Create
insertProduct := &Product{Code: "D42", Price: 100}
db.Create(insertProduct)
fmt.Println(insertProduct.ID)
// Read
var product Product
tx := db.First(&product, 1) // find product with integer primary key
if tx.Error != nil {
fmt.Println("not found first ", tx.Error.Error())
}
db.First(&product, "code = ?", "D42")
// Delete - delete product
db.Delete(&product, 1)
}
func testClickhouse() {
err := gogorm.Init(gogorm.Config{
DataSource: "tcp://localhost:9000?database=gorm&username=gorm&password=gorm&read_timeout=10&write_timeout=20",
DBType: gogorm.DATABASE_CLICKHOUSE,
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.Set("gorm:table_options", "ENGINE=Distributed(cluster, default, hits)").AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
// Set table options
// Create
insertProduct := &Product{Code: "D42", Price: 100}
db.Create(insertProduct)
fmt.Println(insertProduct.ID)
// Read
var product Product
tx := db.First(&product, 1) // find product with integer primary key
if tx.Error != nil {
fmt.Println("not found first ", tx.Error.Error())
}
db.First(&product, "code = ?", "D42")
// Delete - delete product
db.Delete(&product, 1)
}
func testTransaction() {
err := gogorm.Init(gogorm.Config{
DataSource: "root:223238@tcp(127.0.0.1:33060)/gromdb?charset=utf8mb4&parseTime=True&loc=Local",
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
tx := db.Begin()
// Create
insertProduct := &Product{Code: "D42", Price: 100}
txd := tx.Create(insertProduct)
if txd.Error != nil {
fmt.Println("Insert error ", txd.Error.Error())
tx.Rollback()
return
}
newId := insertProduct.ID
fmt.Println("Inserted ID: " + gconv.String(newId))
// Read
var product Product
txd = tx.First(&product, insertProduct.ID) // find product with integer primary key
if txd.Error != nil {
fmt.Println("not found first error ", txd.Error.Error())
tx.Rollback()
return
}
txd = tx.First(&product, "code = ?", "D42")
if txd.Error != nil {
fmt.Println("not found first error1 ", txd.Error.Error())
tx.Rollback()
return
}
deleteProduct := Product{}
// Delete - delete product
deleteProduct.ID = newId
txd = tx.Delete(&deleteProduct)
if txd.Error != nil {
fmt.Println("Delete error ", txd.Error.Error())
tx.Rollback()
}
tx.Commit()
}
func testHasMany() {
// User 有多张 CreditCard,UserID 是外键
type CreditCard struct {
gorm.Model
Number string
UserRefer uint
}
type User struct {
gorm.Model
CreditCards []CreditCard `gorm:"foreignKey:UserRefer"`
}
err := gogorm.Init(gogorm.Config{
DataSource: "root:223238@tcp(127.0.0.1:33060)/gromdb?charset=utf8mb4&parseTime=True&loc=Local",
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.AutoMigrate(&User{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
user := User{
CreditCards: []CreditCard{
CreditCard{Number: "jinzhu"},
CreditCard{Number: "jinzhu"},
},
}
db.Create(&user)
//db.Save(&user)
// 检索用户列表并预加载信用卡
var users []User
err = db.Model(&User{}).Preload("CreditCard").Find(&users).Error
if err != nil {
golog.WithTag("godb").Error("检索用户列表并预加载信用卡:" + err.Error())
} else {
fmt.Println(users)
}
}
func postgreSqlTest() {
err := gogorm.Init(gogorm.Config{
DataSource: "host=122.228.113.238 user=postgres password=223238 dbname=passwall port=5432 sslmode=disable TimeZone=Asia/Shanghai",
DBType: gogorm.DATABASE_POSTGRESQL,
})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
db := gogorm.Default().DB
err = db.AutoMigrate(&Product{})
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
// Create
insertProduct := &Product{Code: "D42", Price: 100}
db.Create(insertProduct)
fmt.Println(insertProduct.ID)
// Read
var product Product
tx := db.First(&product, 1) // find product with integer primary key
if tx.Error != nil {
fmt.Println("not found first ", tx.Error.Error())
}
db.First(&product, "code = ?", "D42")
// Delete - delete product
db.Delete(&product, 1)
}
func (u *Product) BeforeDelete(tx *gorm.DB) (err error) {
//if u.Role == "admin" {
// return errors.New("admin user not allowed to delete")
//}
fmt.Println("BeforeDelete ")
return nil
}
func (u *Product) BeforeCreate(tx *gorm.DB) (err error) {
//if u.Role == "admin" {
// return errors.New("admin user not allowed to delete")
//}
fmt.Println("BeforeCreate ")
return nil
}
go-doc/Event 结构体字段说明文档.md (7.4 KiB)
Event 结构体字段说明文档
基础信息字段
| 字段名 | 类型 | 说明 |
|---|---|---|
| AppCode | string | 产品 code |
| UserId | string | 用户 id |
| RegisterDate | string | 用户注册日期 |
| RegisterTime | time.Time | 用户注册时间戳(秒) |
请求相关字段
| 字段名 | 类型 | 说明 |
|---|---|---|
| RequestTime | time.Time | 请求时间戳,使用客户端 time 或 LogTime(两者差值小于 30 分钟时使用客户端 time,否则使用 LogTime) |
| RequestDate | string | 请求日期(2025-01-11) |
| Hour | uint8 | 当前小时 |
设备和系统信息
| 字段名 | 类型 | 说明 |
|---|---|---|
| OsType | string | 设备类型(mobile-ios, mobile-android, desktop-windows, desktop-macos) |
| SystemVersion | string | 系统版本号 |
| Brand | string | 手机型号(参考:文档最后 设备列表) |
| DeviceId | string | 设备 ID |
| GoogleId | string | Google ID |
| AndroidId | string | Android ID |
渠道和广告系列信息
| 字段名 | 类型 | 说明 |
|---|---|---|
| Channel | string | 渠道 |
| CampaignId | string | 系列 Id |
| CampaignName | string | 系列名 |
| AdCostMode | string | 广告计费模型 |
| CampaignChannel | string | 系列渠道(如 fb) |
| CampaignPartner | string | 系列合作人 |
地理位置和网络信息
| 字段名 | 类型 | 说明 |
|---|---|---|
| Ip | string | 广告的 IP(IsVpn=1 时为 漂移国家ip,否则为 用户国家 ip) |
| VpnIp | string | VPN 节点 IP |
| VpnIsp | string | VPN IP 对应的运营商(使用客户端上报的 VpnIdc) |
| GeoIp | string | 地理位置 IP |
| GeoCountry | string | 用户国家(通过 API 获取) |
| GeoIsp | string | 地理位置运营商 |
| City | string | IP 对应城市(使用客户端上报的 VpnRegion代替) |
| VpnIdc | string | VPN 节点所在的 IDC 机房或运营商 |
| VpnRegion | string | VPN 节点所在的地区 |
事件相关字段
| 字段名 | 类型 | 说明 |
|---|---|---|
| EventMode | string | 事件类型(client, server) |
| Event | string | 事件枚举值(ad_show, ad_click, ad_error, ad_req_success) |
| EventMsg | string | 事件信息(如 ad_error 的详细错误信息) |
| ErrorType | string | 错误类型 |
版本和状态标识
| 字段名 | 类型 | 说明 |
|---|---|---|
| Vc | uint16 | APP 版本号(如:1) |
| Vn | string | APP 版本名(如:1.0.0) |
| IsVpn | uint8 | 是否使用 VPN 请求 |
| IsVpnAd | uint8 | 是否为 VPN 广告 |
广告相关字段
| 字段名 | 类型 | 说明 |
|---|---|---|
| Ecpm | float64 | 每次展示的广告价值(没有时用 0) |
| Platform | string | 广告平台(如 AdMob,Meta, Applovin, Pangle 等) |
| AdType | string | 广告位类型(app_open/rewarded_video/interstitial/banner/native) |
| AdCountryCode | string | 广告的国家(漂移国家-小写国家code,如:美国-》us) |
| PlatformAccountCode | string | 广告平台账号 code |
| PlatformAccountName | string | 广告平台账号名称 |
| PlatformAccountAppId | string | 平台账号下的 AppId |
| AdPlacementCode | string | 广告位 code |
| AdPlacementName | string | 广告位名称 |
| AdSegmentCode | string | 广告分组 code |
| AdSegmentName | string | 广告分组名称 |
| MediaPlatform | string | 第三方聚合平台(如 TopOn, Max 等) |
| MediaSlotId | string | 第三方聚合平台广告位 ID(TopOnID) |
| SlotId | string | 实际的广告位 ID(如 AdMob 的广告位 ID) |
| LoadTime | int64 | 请求加载时间(单位:秒) |
| PagePos | string | 页面位置 |
| CacheAd | uint8 | 缓存广告标识 |
| TimeSlot60 | uint8 | 小时60分钟纬度 | | AdShowSuccessCount | uint8 | 广告展示 | | AvgIpAdShowSuccessCount | uint8 | 广告展示/每Ip | | AdIpCount | uint8 | 独立IP数量 |
其他字段
| 字段名 | 类型 | 说明 |
|---|---|---|
| Days | int64 | 留存天数 (如:Days=1 是新用户) |
| LastNodeId | string | 节点 ID |
| CurrencyCode | string | 货币代码 |
| LogTime | time.Time | 日志服务器收到日志的时间 |
| IpSource | string | IP 来源 |
| LogId | string | 日志 ID |
| MergeVersion | uint32 | 日志合并版本号(值为:MaxUint32-RequestTime,目的:合并时使用 RequestTime 最小值) |
自定义相关字段
| 字段名 | 类型 | 说明 |
|---|---|---|
| TimeSlot60 | uint8 | 小时60分钟纬度 |
| AdShowSuccessCount | uint8 | 广告展示 |
| AvgIpAdShowSuccessCount | uint8 | 广告展示/每Ip |
| AdIpCount | uint8 | 独立IP数量 |
| AdLastNodeIdCount | uint8 | 独立节点数量 |
| AvgIpAdUserCount | uint8 | 独立用户/每Ip |
| NewUserCount | uint8 | 新用户数 |
| AdUserIpu | uint8 | 广告数量/用户 |
| AdUserCount | uint8 | 广告用户 |
| AvgEcpm | uint8 | 平均Ecpm |
| income | uint8 | 收入 |
- 设备列表
var AppleDevicesNames = []string{
"Simulator",
"i386",
"iPad",
"iPad 10",
"iPad 2",
"iPad 3",
"iPad 3G",
"iPad 4",
"iPad 5",
"iPad 6",
"iPad 7",
"iPad 8",
"iPad 9",
"iPad Air",
"iPad Air 2",
"iPad Air 3",
"iPad Air 4",
"iPad Air 5",
"iPad Air 6th Gen",
"iPad Mini",
"iPad Mini 2",
"iPad Mini 3",
"iPad Mini 4",
"iPad Mini 5",
"iPad Mini 6",
"iPad Pro 10.5 inch",
"iPad Pro 11-inch",
"iPad Pro 11-inch 2nd gen",
"iPad Pro 11-inch 3nd gen",
"iPad Pro 11-inch 4th gen",
"iPad Pro 11-inch 5th Gen",
"iPad Pro 12.9",
"iPad Pro 12.9 inch 2nd gen",
"iPad Pro 12.9-inch 3rd gen",
"iPad Pro 12.9-inch 4th gen",
"iPad Pro 12.9-inch 5th gen",
"iPad Pro 12.9-inch 6th gen",
"iPad Pro 12.9-inch 7th Gen",
"iPad Pro 9.7",
"iPhone 11",
"iPhone 11 Pro",
"iPhone 11 Pro Max",
"iPhone 12",
"iPhone 12 Pro",
"iPhone 12 Pro Max",
"iPhone 12 mini",
"iPhone 13",
"iPhone 13 Pro",
"iPhone 13 Pro Max",
"iPhone 13 mini",
"iPhone 14",
"iPhone 14 Plus",
"iPhone 14 Pro",
"iPhone 14 Pro Max",
"iPhone 15",
"iPhone 15 Plus",
"iPhone 15 Pro",
"iPhone 15 Pro Max",
"iPhone 16",
"iPhone 16 Plus",
"iPhone 16 Pro",
"iPhone 16 Pro Max",
"iPhone 4",
"iPhone 4S",
"iPhone 5",
"iPhone 5c",
"iPhone 5s",
"iPhone 6",
"iPhone 6 Plus",
"iPhone 6s",
"iPhone 6s Plus",
"iPhone 7",
"iPhone 7 Plus",
"iPhone 8",
"iPhone 8 Plus",
"iPhone SE",
"iPhone SE 2",
"iPhone SE 3",
"iPhone X",
"iPhone XR",
"iPhone XS",
"iPhone XS Max",
"iPod Touch 1G",
"iPod Touch 2G",
"iPod Touch 3G",
"iPod Touch 4G",
"iPod Touch 5G",
"iPod Touch 6G",
"iPod Touch 7G",
"x86_64",
}
go-error/errCode.go (921 B)
package goerror
// 常用错误码
// 成功返回
const OK uint32 = 0
/**(前3位代表业务,后三位代表具体功能)**/
// 全局错误码
// client
const REQUEST_PARAM_ERROR uint32 = 400
const TOKEN_EXPIRE_ERROR uint32 = 401
const FORBIDDEN_ERROR uint32 = 403
const NOT_FOUND_ERROR uint32 = 404
const SERVER_COMMON_ERROR uint32 = 500
const CAPTCHA_ERROR uint32 = 700
const USER_NOT_EXISTS_ERROR uint32 = 701
const USER_LOGIN_ERROR uint32 = 703
const USER_EXISTS_ERROR uint32 = 704
// server
const DB_ERROR uint32 = 555
const REDIS_ERROR uint32 = 666
const ETCD_ERROR uint32 = 777
// 错误处理枚举
// showType?: number; // error display type: 0 silent; 1 message.warn; 2 message.error; 4 notification; 9 page
const ShowTypeSilent uint32 = 0
const ShowTypeMessageWarn uint32 = 1
const ShowTypeMessageError uint32 = 2
const ShowTypeNotification uint32 = 4
const ShowTypePage uint32 = 9 // 页面跳转
go-error/errMsg.go (1.7 KiB)
package goerror
var (
errorsx = map[uint32]string{}
message = map[uint32]string{
OK: "ok",
SERVER_COMMON_ERROR: "server error",
NOT_FOUND_ERROR: "not found error",
REQUEST_PARAM_ERROR: "param error",
TOKEN_EXPIRE_ERROR: "token expired,please login again",
USER_EXISTS_ERROR: "user exists",
USER_NOT_EXISTS_ERROR: "user not exists",
USER_LOGIN_ERROR: "username or password is invalid",
FORBIDDEN_ERROR: "forbidden",
DB_ERROR: "db error",
CAPTCHA_ERROR: "captcha error",
}
)
var (
ErrNotFound = NewErrCodeMsg(NOT_FOUND_ERROR, "not found")
ErrUserNoExists = NewErrCodeMsg(USER_NOT_EXISTS_ERROR, "user not found")
ErrUserExists = NewErrCodeMsg(USER_EXISTS_ERROR, "user exists")
ErrUserForbidden = NewErrCode(FORBIDDEN_ERROR)
ErrUserLogin = NewErrCode(USER_LOGIN_ERROR)
ErrUnauthorized = NewErrCodeMsg(TOKEN_EXPIRE_ERROR, "unauthorized")
ErrCaptcha = NewErrCodeMsg(CAPTCHA_ERROR, "captcha error")
ErrRequestParamError = NewErrCode(REQUEST_PARAM_ERROR)
ErrDBError = NewErrCode(DB_ERROR)
ErrServerError = NewErrCode(SERVER_COMMON_ERROR)
ErrRedisError = NewErrCode(REDIS_ERROR)
ErrEtcdError = NewErrCode(ETCD_ERROR)
)
// 扩展的错误类型这里初始化, GetErrCodeMsg 中自动处理
// 优先使用自定义错误类型,然后再使用扩展错误类型
func Init(errs map[uint32]string) {
errorsx = errs
}
func MapErrMsg(errcode uint32) string {
if msg, ok := message[errcode]; ok {
return msg
} else {
return "server error"
}
}
func IsCodeErr(errcode uint32) bool {
if _, ok := message[errcode]; ok {
return true
} else {
return false
}
}
go-error/errors.go (3.9 KiB)
package goerror
import (
"fmt"
)
// 常用通用固定错误
//
// codeError := NewCodeErrorBuilder().
// WithErrCode(1001).
// WithErrMsg("系统错误").
// WithShowType(1).
// WithTraceId("trace-123").
// WithHost("localhost").
// Build()
type CodeError struct {
errCode uint32
errMsg string
showType uint32
traceId string
host string
}
// CodeErrorBuilder 是 CodeError 的构建器
type CodeErrorBuilder struct {
error *CodeError
}
// 返回给前端的错误码
func (e *CodeError) GetErrCode() uint32 {
return e.errCode
}
// 返回给前端显示端错误信息
func (e *CodeError) GetErrMsg() string {
return e.errMsg
}
func (e *CodeError) GetShowType() uint32 {
return e.showType
}
func (e *CodeError) GetTraceId() string {
return e.traceId
}
func (e *CodeError) GetHost() string {
return e.host
}
func (e *CodeError) Error() string {
//return fmt.Sprintf("ErrCode:%d,ErrMsg:%s", e.errCode, e.errMsg)
return fmt.Sprintf("ErrCode:%d,ErrMsg:%s,ShowType:%d,TraceId:%s,Host:%s", e.errCode, e.errMsg, e.showType, e.traceId, e.host)
}
func NewErrCodeMsg(errCode uint32, errMsg string) *CodeError {
builder := NewCodeErrorBuilder()
builder.WithErrCode(errCode).WithErrMsg(errMsg)
return builder.Build()
}
func NewErrCodeMsgForMessageError(errCode uint32, errMsg string) *CodeError {
builder := NewCodeErrorBuilder()
builder.WithErrCode(errCode).WithErrMsg(errMsg).WithShowType(ShowTypeMessageError)
return builder.Build()
}
func NewErrCodeMsgForNotification(errCode uint32, errMsg string) *CodeError {
builder := NewCodeErrorBuilder()
builder.WithErrCode(errCode).WithErrMsg(errMsg).WithShowType(ShowTypeNotification)
return builder.Build()
}
func NewErrCodeMsgForMessageWarn(errCode uint32, errMsg string) *CodeError {
builder := NewCodeErrorBuilder()
builder.WithErrCode(errCode).WithErrMsg(errMsg).WithShowType(ShowTypeMessageWarn)
return builder.Build()
}
func NewErrCodeMsgForPage(errCode uint32, errMsg string) *CodeError {
builder := NewCodeErrorBuilder()
builder.WithErrCode(errCode).WithErrMsg(errMsg).WithShowType(ShowTypePage)
return builder.Build()
}
func NewErrCode(errCode uint32) *CodeError {
builder := NewCodeErrorBuilder()
builder.WithErrCode(errCode).WithErrMsg(MapErrMsg(errCode))
return builder.Build()
}
func NewErrMsg(errMsg string) *CodeError {
builder := NewCodeErrorBuilder()
builder.WithErrCode(SERVER_COMMON_ERROR).WithErrMsg(errMsg)
return builder.Build()
}
func NewParamErrMsg(errMsg string) *CodeError {
builder := NewCodeErrorBuilder()
builder.WithErrCode(REQUEST_PARAM_ERROR).WithErrMsg(errMsg)
return builder.Build()
}
func NewErrorMsg(errCode uint32, errMsg string) error {
builder := NewCodeErrorBuilder()
builder.WithErrCode(errCode).WithErrMsg(errMsg)
return builder.Build()
}
func NewError(errCode uint32) error {
builder := NewCodeErrorBuilder()
builder.WithErrCode(errCode).WithErrMsg(MapErrMsg(errCode))
return builder.Build()
}
// NewCodeErrorBuilder 创建一个新的 CodeErrorBuilder
func NewCodeErrorBuilder() *CodeErrorBuilder {
return &CodeErrorBuilder{
error: &CodeError{},
}
}
// WithErrCode 设置错误码
func (b *CodeErrorBuilder) WithErrCode(errCode uint32) *CodeErrorBuilder {
b.error.errCode = errCode
return b
}
// WithErrMsg 设置错误信息
func (b *CodeErrorBuilder) WithErrMsg(errMsg string) *CodeErrorBuilder {
b.error.errMsg = errMsg
return b
}
// WithShowType 设置显示类型
func (b *CodeErrorBuilder) WithShowType(showType uint32) *CodeErrorBuilder {
b.error.showType = showType
return b
}
// WithTraceId 设置追踪ID
func (b *CodeErrorBuilder) WithTraceId(traceId string) *CodeErrorBuilder {
b.error.traceId = traceId
return b
}
// WithHost 设置主机信息
func (b *CodeErrorBuilder) WithHost(host string) *CodeErrorBuilder {
b.error.host = host
return b
}
// Build 构建并返回 CodeError 实例
func (b *CodeErrorBuilder) Build() *CodeError {
return b.error
}
go-error/httpCode.go (1.9 KiB)
package goerror
import "fmt"
type HttpCodeError struct {
errCode uint32
errMsg string
showType uint32
traceId string
host string
}
// 返回给前端的错误码
func (e *HttpCodeError) GetErrCode() uint32 {
return e.errCode
}
// 返回给前端显示端错误信息
func (e *HttpCodeError) GetErrMsg() string {
return e.errMsg
}
func (e *HttpCodeError) GetShowType() uint32 {
return e.showType
}
func (e *HttpCodeError) GetTraceId() string {
return e.traceId
}
func (e *HttpCodeError) GetHost() string {
return e.host
}
func (e *HttpCodeError) Error() string {
//return fmt.Sprintf("ErrCode:%d,ErrMsg:%s", e.errCode, e.errMsg)
return fmt.Sprintf("ErrCode:%d,ErrMsg:%s,ShowType:%d,TraceId:%s,Host:%s", e.errCode, e.errMsg, e.showType, e.traceId, e.host)
}
type HttpCodeErrorBuilder struct {
error *HttpCodeError
}
// NewHttpCodeErrorBuilder 创建一个新的 HttpCodeErrorBuilder
func NewHttpCodeErrorBuilder() *HttpCodeErrorBuilder {
return &HttpCodeErrorBuilder{
error: &HttpCodeError{},
}
}
// WithErrCode 设置错误码
func (b *HttpCodeErrorBuilder) WithErrCode(errCode uint32) *HttpCodeErrorBuilder {
b.error.errCode = errCode
return b
}
// WithErrMsg 设置错误信息
func (b *HttpCodeErrorBuilder) WithErrMsg(errMsg string) *HttpCodeErrorBuilder {
b.error.errMsg = errMsg
return b
}
// WithShowType 设置显示类型
func (b *HttpCodeErrorBuilder) WithShowType(showType uint32) *HttpCodeErrorBuilder {
b.error.showType = showType
return b
}
// WithTraceId 设置追踪ID
func (b *HttpCodeErrorBuilder) WithTraceId(traceId string) *HttpCodeErrorBuilder {
b.error.traceId = traceId
return b
}
// WithHost 设置主机信息
func (b *HttpCodeErrorBuilder) WithHost(host string) *HttpCodeErrorBuilder {
b.error.host = host
return b
}
// Build 构建并返回 CodeError 实例
func (b *HttpCodeErrorBuilder) Build() *HttpCodeError {
return b.error
}
go-error/httpErrCode.go (502 B)
package goerror
const STATUS_OK uint32 = 200
const STATUS_NOCONTENT uint32 = 204
const STATUS_FOUND uint32 = 302
const STATUS_UNAUTHORIZATION uint32 = 401
const STATUS_FORBIDDEN_ERROR uint32 = 403
const STATUS_NOT_FOUND_ERROR uint32 = 404
const STATUS_PARAM_ERROR uint32 = 400
const STATUS_LOCK_ERROR uint32 = 423
const STATUS_FAILED_ERROR uint32 = 424
const STATUS_INTERNAL_SERVER_ERROR uint32 = 500
const STATUS_SERVICE_UNAVAILABLE_ERROR uint32 = 503
const STATUS_GATEWAY_TIMEOUT_ERROR uint32 = 504
go-error/readme.md (625 B)
有状态信息的错误封装
package main
import (
goerror "github.com/gif-gif/go.io/go-error"
golog "github.com/gif-gif/go.io/go-log"
"github.com/pkg/errors"
)
func main() {
err := errors.Wrapf(goerror.NewErrCode(goerror.DB_ERROR), "find customer db err, in:%v , err:%v", "args", "err")
if goerror.IsCodeErr(goerror.DB_ERROR) {
golog.WithTag("goerror1").Error(err.Error(), " db error")
}
if goerror.IsErrCode(err, goerror.DB_ERROR) {
golog.WithTag("goerror2").Error(err.Error(), " db error")
}
errCode, errMsg := goerror.GetErrCodeMsg(err)
golog.WithTag("goerror3").Error(errCode, errMsg)
}
go-error/test/main.go (580 B)
package main
import (
goerror "github.com/gif-gif/go.io/go-error"
golog "github.com/gif-gif/go.io/go-log"
"github.com/pkg/errors"
)
func main() {
err := errors.Wrapf(goerror.NewErrCode(goerror.DB_ERROR), "find customer db err, in:%v , err:%v", "args", "err")
if goerror.IsCodeErr(goerror.DB_ERROR) {
golog.WithTag("goerror1").Error(err.Error(), " db error")
}
if goerror.IsErrCode(err, goerror.DB_ERROR) {
golog.WithTag("goerror2").Error(err.Error(), " db error")
}
errCode, errMsg := goerror.GetErrCodeMsg(err)
golog.WithTag("goerror3").Error(errCode, errMsg)
}
go-error/util.go (1.5 KiB)
package goerror
import (
"github.com/pkg/errors"
"google.golang.org/grpc/status"
"runtime"
)
func IsDatabaseNoRowsError(err error) bool {
return err != nil && err.Error() == "sql: no rows in result set"
}
func GetStack() string {
var buf [4096]byte
n := runtime.Stack(buf[:], false)
return string(buf[:n])
}
func IsErrCode(err error, code uint32) bool {
errCode, _ := GetErrCodeMsg(err)
return code == errCode
}
// 建议用 GetCodeError
func GetErrCodeMsg(err error) (errCode uint32, errMsg string) {
if err == nil {
return 0, ""
}
errCode = SERVER_COMMON_ERROR
errMsg = "server error"
causeErr := errors.Cause(err) // err类型
if e, ok := causeErr.(*CodeError); ok { //自定义错误类型
errCode = e.GetErrCode()
errMsg = e.GetErrMsg()
} else if er, ok := err.(*HttpCodeError); ok { //http 状态码
errCode = er.GetErrCode()
errMsg = er.GetErrMsg()
} else { //通用错误
errCode, errMsg = GetErrorMsg(err)
}
return
}
func GetErrorMsg(err error) (uint32, string) {
causeErr := errors.Cause(err)
if gstatus, ok := status.FromError(causeErr); ok { // grpc err错误
grpcCode := uint32(gstatus.Code())
return grpcCode, gstatus.Message()
}
return 500, "server error"
}
// CodeErrorBuilder.build() 构建 CodeError
//
// 返回错误码 CodeErrorBuilder
func GetCodeError(err error) *CodeErrorBuilder {
codeErr := NewCodeErrorBuilder()
if err == nil {
return codeErr
}
errCode, errMsg := GetErrCodeMsg(err)
codeErr.WithErrCode(errCode).WithErrMsg(errMsg)
return codeErr
}
go-etcd/client.go (7.7 KiB)
package goetcd
import (
"context"
"crypto/tls"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"go.etcd.io/etcd/client/pkg/v3/transport"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/naming/endpoints"
"go.uber.org/zap"
"runtime"
"strconv"
"sync"
"time"
)
type GoEtcdClient struct {
*clientv3.Client
ctx context.Context
conf Config
}
func New(conf Config) (cli *GoEtcdClient, err error) {
cli = &GoEtcdClient{ctx: context.TODO(), conf: conf}
defer func() {
if err != nil {
time.Sleep(5 * time.Second)
cli, _ = New(conf)
}
}()
cfg := clientv3.Config{
Endpoints: conf.Endpoints,
DialTimeout: 5 * time.Second,
Logger: zap.NewNop(),
}
if conf.Username != "" {
cfg.Username = conf.Username
}
if conf.Password != "" {
cfg.Password = conf.Password
}
if conf.TLS != nil {
tlsInfo := &transport.TLSInfo{
CertFile: conf.TLS.CertFile,
KeyFile: conf.TLS.KeyFile,
TrustedCAFile: conf.TLS.CAFile,
}
var clientConfig *tls.Config
clientConfig, err = tlsInfo.ClientConfig()
if err != nil {
golog.WithTag("go-etcd").WithField("config", conf).Error(err.Error())
return
}
cfg.TLS = clientConfig
}
cli.Client, err = clientv3.New(cfg)
if err != nil {
golog.WithTag("go-etcd").WithField("config", conf).Error(err.Error())
}
return
}
// set key-value
func (cli *GoEtcdClient) Set(key, val string, opts ...clientv3.OpOption) (resp *clientv3.PutResponse, err error) {
resp, err = cli.Client.Put(cli.ctx, key, val, opts...)
if err != nil {
golog.WithTag("go-etcd").WithField("key", key).WithField("val", val).Error(err)
}
return
}
// set key-value and return previous key-value
func (cli *GoEtcdClient) SetWithPrevKV(key, val string) (resp *clientv3.PutResponse, err error) {
return cli.Set(key, val, clientv3.WithPrevKV())
}
// set key-value-ttl
func (cli *GoEtcdClient) SetTTL(key, val string, ttl int64, opts ...clientv3.OpOption) (resp *clientv3.PutResponse, err error) {
if ttl == 0 {
return cli.Set(key, val, opts...)
}
var lease *clientv3.LeaseGrantResponse
lease, err = cli.Client.Grant(cli.ctx, ttl)
if err != nil {
golog.WithTag("go-etcd").WithField("key", key).WithField("val", val).WithField("ttl", ttl).Error(err)
return
}
_, err = cli.Client.Put(cli.ctx, key, val, clientv3.WithLease(lease.ID))
if err != nil {
golog.WithTag("go-etcd").WithField("key", key).WithField("val", val).WithField("ttl", ttl).Error(err)
return
}
return
}
// set key-value-ttl and return previous key-value
func (cli *GoEtcdClient) SetTTLWithPrevKV(key, val string, ttl int64) (resp *clientv3.PutResponse, err error) {
return cli.SetTTL(key, val, ttl, clientv3.WithPrevKV())
}
// get value by key
func (cli *GoEtcdClient) Get(key string, opts ...clientv3.OpOption) (resp *clientv3.GetResponse, err error) {
resp, err = cli.Client.Get(cli.ctx, key, opts...)
if err != nil {
golog.WithTag("go-etcd").WithField("key", key).Error(err)
}
return
}
// get string value by prefix key
func (cli *GoEtcdClient) GetString(key string) string {
resp, err := cli.Get(key, clientv3.WithPrefix())
if err != nil {
return ""
}
if l := len(resp.Kvs); l == 0 {
return ""
}
return string(resp.Kvs[0].Value)
}
// get array value by prefix key
func (cli *GoEtcdClient) GetArray(key string) (data []string) {
data = []string{}
resp, err := cli.Get(key, clientv3.WithPrefix())
if err != nil {
golog.WithTag("go-etcd").WithField("key", key).Error(err)
return
}
for _, i := range resp.Kvs {
data = append(data, string(i.Value))
}
return
}
// get map value by prefix key
func (cli *GoEtcdClient) GetMap(key string) (data map[string]string) {
data = map[string]string{}
resp, err := cli.Get(key, clientv3.WithPrefix())
if err != nil {
golog.WithTag("go-etcd").WithField("key", key).Error(err)
return
}
for _, i := range resp.Kvs {
key := string(i.Key)
data[key] = string(i.Value)
}
return
}
// del key and return previous key-value
func (cli *GoEtcdClient) Del(key string, opts ...clientv3.OpOption) (resp *clientv3.DeleteResponse, err error) {
opts = append(opts, clientv3.WithPrevKV())
resp, err = cli.Client.Delete(cli.ctx, key, opts...)
if err != nil {
golog.WithTag("go-etcd").WithField("key", key).Error(err)
}
return
}
// del prefix key and return previous key-value
func (cli *GoEtcdClient) DelWithPrefix(key string) (resp *clientv3.DeleteResponse, err error) {
return cli.Del(key, clientv3.WithPrefix())
}
// register service and keepalive
func (cli *GoEtcdClient) RegisterService(serviceName, addr string) (err error) {
defer func() {
if cli.Client == nil || err != nil {
time.Sleep(3 * time.Second)
cli.RegisterService(serviceName, addr)
}
}()
var (
ttl int64 = 5
em endpoints.Manager
lease *clientv3.LeaseGrantResponse
ch <-chan *clientv3.LeaseKeepAliveResponse
)
lease, err = cli.Client.Grant(cli.ctx, ttl)
if err != nil {
golog.WithTag("go-etcd").WithField("serviceName", serviceName).WithField("addr", addr).Error(err)
return
}
em, err = endpoints.NewManager(cli.Client, serviceName)
if err != nil {
golog.WithTag("go-etcd").WithField("serviceName", serviceName).WithField("addr", addr).Error(err)
return
}
serviceKey := serviceName + "/" + strconv.Itoa(int(lease.ID))
err = em.AddEndpoint(cli.ctx, serviceKey, endpoints.Endpoint{Addr: addr}, clientv3.WithLease(lease.ID))
if err != nil {
golog.WithTag("go-etcd").WithField("serviceName", serviceName).WithField("addr", addr).Error(err)
return
}
ch, err = cli.Client.KeepAlive(cli.ctx, lease.ID)
if err != nil {
golog.WithTag("go-etcd").WithField("serviceName", serviceName).WithField("addr", addr).Error(err)
return
}
golog.WithTag("go-etcd").WithField("serviceName", serviceName).WithField("addr", addr).Debug("服务注册成功")
go func() {
for {
select {
case <-cli.ctx.Done():
golog.WithTag("go-etcd").WithField("serviceName", serviceName).WithField("addr", addr).Warn("服务退出,收回注册信息")
cli.Client.Revoke(cli.ctx, lease.ID)
return
case rsp := <-ch:
if rsp == nil {
golog.WithTag("go-etcd").WithField("serviceName", serviceName).WithField("addr", addr).Error("服务注册续租失效")
cli.RegisterService(serviceName, addr)
return
}
}
}
}()
return
}
// watch the key
func (cli *GoEtcdClient) Watch(key string) <-chan []string {
var (
mu sync.Mutex
ch = make(chan []string, runtime.NumCPU()*2)
data = cli.GetMap(key)
)
ch <- cli.map2array(data)
go func() {
defer func() {
if r := recover(); r != nil {
golog.WithTag("go-etcd").WithField("key", key).Error(r)
}
}()
wc := cli.Client.Watch(cli.ctx, key, clientv3.WithPrefix())
for {
select {
case <-cli.ctx.Done():
return
case w := <-wc:
mu.Lock()
for _, ev := range w.Events {
k := string(ev.Kv.Key)
v := string(ev.Kv.Value)
switch ev.Type {
case clientv3.EventTypePut:
data[k] = v
case clientv3.EventTypeDelete:
delete(data, k)
}
}
ch <- cli.map2array(data)
mu.Unlock()
}
}
}()
return ch
}
func (cli *GoEtcdClient) map2array(data map[string]string) []string {
var arrData []string
for _, v := range data {
arrData = append(arrData, v)
}
return arrData
}
// 检测 rpc服务是否启动
func (cli *GoEtcdClient) CheckRpcServices(rpcServices []string) bool {
rpcStarted := true
for _, service := range rpcServices {
rpc := GetMap(service)
if len(rpc) == 0 {
if goio.Env == goio.TEST || goio.Env == goio.PRODUCTION {
golog.WithTag("CheckRpcService").Fatal(service + " rpc is not started")
} else {
golog.WithTag("CheckRpcService").Error(service + " rpc is not started")
}
rpcStarted = false
} else {
golog.WithTag("CheckRpcService").Info(service + " rpc is started")
}
}
return rpcStarted
}
go-etcd/config.go (541 B)
package goetcd
type Config struct {
Endpoints []string `json:"endpoints,optional" yaml:"Endpoints"`
Name string `json:"name,optional" yaml:"Name"`
Username string `json:"username,optional" yaml:"Username"`
Password string `json:"password,optional" yaml:"Password"`
TLS *TLS `json:"tls,optional" yaml:"TLS"`
}
type TLS struct {
CertFile string `json:"certFile,optional" yaml:"CertFile"`
KeyFile string `json:"keyFile,optional" yaml:"KeyFile"`
CAFile string `json:"caFile,optional" yaml:"CAFile"`
}
go-etcd/etcd.go (2.4 KiB)
package goetcd
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
clientv3 "go.etcd.io/etcd/client/v3"
)
var __clients = map[string]*GoEtcdClient{}
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name], err = New(conf)
if err != nil {
return
}
}
return
}
func GetClient(names ...string) *GoEtcdClient {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoEtcdClient {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("goetcd").Error("no default goetcd client")
return nil
}
func Set(key, val string) (resp *clientv3.PutResponse, err error) {
return Default().Set(key, val)
}
func SetWithPrevKV(key, val string) (resp *clientv3.PutResponse, err error) {
return Default().SetWithPrevKV(key, val)
}
func SetTTL(key, val string, ttl int64) (resp *clientv3.PutResponse, err error) {
return Default().SetTTL(key, val, ttl)
}
func SetTTLWithPrevKV(key, val string, ttl int64) (resp *clientv3.PutResponse, err error) {
return Default().SetTTLWithPrevKV(key, val, ttl)
}
func Get(key string, opts ...clientv3.OpOption) (rsp *clientv3.GetResponse, err error) {
return Default().Get(key, opts...)
}
func Exists(key string, opts ...clientv3.OpOption) (bool, error) {
e, err := Get(key, opts...)
if err != nil {
return false, err
}
return e.Count > 0, err
}
func GetString(key string) string {
return Default().GetString(key)
}
func GetArray(key string) (data []string) {
return Default().GetArray(key)
}
func GetMap(key string) (data map[string]string) {
return Default().GetMap(key)
}
func Del(key string) (resp *clientv3.DeleteResponse, err error) {
return Default().Del(key)
}
func DelWithPrefix(key string) (resp *clientv3.DeleteResponse, err error) {
return Default().DelWithPrefix(key)
}
func RegisterService(key, val string) (err error) {
return Default().RegisterService(key, val)
}
func Watch(key string) <-chan []string {
return Default().Watch(key)
}
go-etcd/etcd_test.go (1.5 KiB)
package goetcd
import (
"fmt"
gocontext "github.com/gif-gif/go.io/go-context"
"log"
"testing"
"time"
)
// 配置结构定义
type TestSt struct {
Name string `json:"name"`
}
func TestInit(t *testing.T) {
Init(Config{
Endpoints: []string{"127.0.0.1:2379"},
//Username: "root",
//Password: "123456",
})
if _, err := Set("/xz/dsp/http-api/192.168.1.101:15001", "192.168.1.101:15001"); err != nil {
log.Fatalln(err)
}
if _, err := Set("/xz/dsp/http-api/192.168.1.101:15002", "192.168.1.101:15002"); err != nil {
log.Fatalln(err)
}
if _, err := SetTTL("/xz/dsp/http-api/192.168.1.101:15003", "192.168.1.101:15003", 3); err != nil {
log.Fatalln(err)
}
for i := 0; i < 2; i++ {
fmt.Println(GetString("/xz/dsp/http-api/"))
fmt.Println(GetArray("/xz/dsp/http-api/"))
fmt.Println(GetMap("/xz/dsp/http-api/"))
time.Sleep(5 * time.Second)
}
Del("/xz/dsp/http-api/192.168.1.101:15002")
}
func TestRegisterService(t *testing.T) {
Init(Config{
Endpoints: []string{"127.0.0.1:23790"},
//Username: "root",
//Password: "123456",
})
err := RegisterService("/xz/dsp/http-api/node-1", "192.168.1.101:15002")
fmt.Println(err)
<-gocontext.WithCancel().Done()
}
func TestWatch(t *testing.T) {
Init(Config{
Endpoints: []string{"127.0.0.1:2379"},
//Username: "root",
//Password: "123456",
})
go func() {
for i := 0; i < 100; i++ {
SetTTL(fmt.Sprintf("/xz/dsp/http-api/node-%d", 100), fmt.Sprintf("192.168.1.%d", i), 5)
time.Sleep(time.Second)
}
}()
ch := Watch("/xz/dsp/http-api")
for i := range ch {
fmt.Println(i)
}
}
go-etcd/readme.md (118 B)
golang operation etcd api
需要实现通过etcd 访问 rpc server服务,不用在go zero 中配置 ---- ing
go-event/README.md (507 B)
go-event 基于 chan
观察者模式 事件中心
扩展开发,定义消息通道大小
//使用方法
package main
import (
goevent "github.com/gif-gif/go.io/go-event"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"time"
)
func main() {
goio.Init()
event := goevent.New()
event.Subscribe("test", func(msg goevent.Message) {
golog.WithTag("goevent").Info(msg)
})
event.Publish("test", "test")
time.Sleep(time.Duration(1) * time.Second)
}
go-event/default.go (370 B)
package goevent
import "sync"
var (
__event *GoEvent
__once sync.Once
)
func Default() *GoEvent {
return __event
}
func Publish(topic string, data interface{}) {
__once.Do(func() {
__event = New()
})
__event.Publish(topic, data)
}
func Subscribe(topic string, fn SubscribeFunc) {
__once.Do(func() {
__event = New()
})
__event.Subscribe(topic, fn)
}
go-event/event.go (2.7 KiB)
package goevent
import (
goutils "github.com/gif-gif/go.io/go-utils"
"sync"
)
// 支持同步和异步执行(发送和接收)
//
// 默认是同步执行
type GoEvent struct {
subscribes map[string][]MessageChan
channelSize int // 消息通道同时处理大小,默认没有限制
mu sync.RWMutex
DefaultTopic string
}
func New(channelSize ...int) *GoEvent {
if len(channelSize) == 0 {
return &GoEvent{subscribes: map[string][]MessageChan{}, channelSize: 0}
} else {
return &GoEvent{subscribes: map[string][]MessageChan{}, channelSize: channelSize[0]}
}
}
// 发布 执行当前topic 对应的所有订阅者, async=true 则异步执行(并发执行无序),否则同步执行保证channel发送顺序
func (ev *GoEvent) Publish(topic string, data interface{}, async ...bool) {
ev.mu.RLock()
defer ev.mu.RUnlock()
if chs, ok := ev.subscribes[topic]; ok {
channels := append([]MessageChan{}, chs...)
if len(async) > 0 && async[0] { //并发执行
goutils.AsyncFunc(func() {
for _, ch := range channels {
ch <- Message{Topic: topic, Data: data}
}
})
} else {
for _, ch := range channels {
ch <- Message{Topic: topic, Data: data}
}
}
}
}
// 订阅:一个topic可以对应多个处理器,(topic->handler 的关系是1:n),一次添加一个订阅者
func (ev *GoEvent) Subscribe(topic string, fn SubscribeFunc, async ...bool) {
ev.mu.Lock()
defer ev.mu.Unlock()
if _, ok := ev.subscribes[topic]; !ok {
ev.subscribes[topic] = []MessageChan{}
}
var ch chan Message
if ev.channelSize == 0 {
ch = make(chan Message)
} else {
ch = make(chan Message, ev.channelSize)
}
if ev.DefaultTopic == "" {
ev.DefaultTopic = topic
}
ev.subscribes[topic] = append(ev.subscribes[topic], ch)
goutils.AsyncFunc(func() {
if len(async) > 0 && async[0] { //并发执行
for msg := range ch {
goutils.AsyncFunc(func() {
fn(msg)
})
}
//v, ok := <-ch
//fmt.Println(v, ok)
} else {
for msg := range ch {
fn(msg)
}
//v, ok := <-ch
//fmt.Println(v, ok)
}
})
}
// 取消订阅(topic 对应的所有订阅者)
func (ev *GoEvent) UnSubscribe(topic string) {
ev.mu.Lock()
defer ev.mu.Unlock()
if chs, ok := ev.subscribes[topic]; ok {
channels := append([]MessageChan{}, chs...)
for _, ch := range channels {
goutils.AsyncFunc(func() {
close(ch)
})
}
delete(ev.subscribes, topic)
}
}
func (ev *GoEvent) UnSubscribeDefault() {
ev.mu.Lock()
defer ev.mu.Unlock()
if chs, ok := ev.subscribes[ev.DefaultTopic]; ok {
channels := append([]MessageChan{}, chs...)
for _, ch := range channels {
goutils.AsyncFunc(func() {
close(ch)
})
}
delete(ev.subscribes, ev.DefaultTopic)
}
}
go-event/message.go (143 B)
package goevent
type Message struct {
Topic string
Data interface{}
}
type MessageChan chan Message
type SubscribeFunc func(msg Message)
go-event/test/main.go (999 B)
package main
import (
gocontext "github.com/gif-gif/go.io/go-context"
goevent "github.com/gif-gif/go.io/go-event"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"github.com/gogf/gf/util/gconv"
"time"
)
type EventField1 struct {
A string
}
type EventField2 struct {
B string
}
type TestEvent struct {
A *EventField1
B *EventField2
}
func main() {
goio.Init(goio.DEVELOPMENT)
simpleTest()
<-gocontext.WithCancel().Done()
}
func simpleTest() {
event := goevent.New()
event.Subscribe("test", func(msg goevent.Message) {
golog.WithTag("goevent").Info(msg)
})
event.Publish("test", "test")
time.Sleep(1 * time.Second)
for i := 0; i < 10; i++ {
event.Publish("test", "test-"+gconv.String(i))
}
event.UnSubscribe("test")
}
func channelSizeTest() {
event := goevent.New(100)
event.Subscribe("test", func(msg goevent.Message) {
golog.WithTag("goevent").Info(msg)
})
for i := 0; i < 100; i++ {
event.Publish("test", "test-"+gconv.String(i))
}
}
go-file/bigfile.go (2.7 KiB)
package gofile
import (
"context"
"errors"
"fmt"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gogf/gf/util/gconv"
"math"
"os"
"path/filepath"
)
type BigFile struct {
ChunkSize int64 // 分片大小 M
MaxWorkers int // 同时处理最大分块数量,合理用防止超大文件内存益处
File string // 文件路径
FileMd5 string // 文件Md5
ChunkCount int64 // 分片数量
FileChunkCallback func(chunk *FileChunk) error // 分片处理消息
SuccessChunkIndexes []int64 //处理成功的碎片index
fileReader *os.File
fileSize int64
ErrorGroup goutils.ErrorGroup
}
func (b *BigFile) IsSuccess() bool {
return len(b.SuccessChunkIndexes) == int(b.ChunkCount)
}
func (b *BigFile) Start() error {
if b.FileChunkCallback == nil {
return errors.New("FileChunkCallback is nil")
}
e, err := Exist(b.File)
if err != nil {
return err
}
if !e {
return errors.New("file not found")
}
file, err := os.Open(b.File)
if err != nil {
return err
}
fileInfo, err := file.Stat()
if err != nil {
return err
}
b.ErrorGroup = goutils.NewErrorGroup(context.TODO(), b.MaxWorkers)
cSize := float64(fileInfo.Size()) / float64(b.ChunkSize*1024*1024)
chunkCount := gconv.Int(math.Ceil(cSize))
b.ChunkCount = int64(chunkCount)
b.fileSize = fileInfo.Size()
b.fileReader = file
for i := 0; i < chunkCount; i++ {
chunkIndex := gconv.Int64(i)
b.ErrorGroup.Submit(func() error {
if b.ErrorGroup.IsContextDone() {
return nil
}
chunk, err := b.createChunk(b.fileReader, chunkIndex)
if err != nil {
return err
}
err = b.FileChunkCallback(chunk)
if err != nil {
return err
}
b.SuccessChunkIndexes = append(b.SuccessChunkIndexes, chunkIndex)
return nil
})
}
return b.ErrorGroup.Wait()
}
func (b *BigFile) createChunk(file *os.File, index int64) (*FileChunk, error) {
bufferSize := b.ChunkSize * 1024 * 1024 // 每次读取MB
startPos := index * bufferSize
buffer := make([]byte, bufferSize)
fileInfo, _ := file.Stat() //剩下不足一个整个bufferSize ,具体的大小计算出来
if index == b.ChunkCount-1 {
buffer = make([]byte, fileInfo.Size()-startPos)
}
_, err := file.ReadAt(buffer, startPos) //TODO: 字节读取验证
if err != nil {
return nil, err
}
hash := goutils.Md5(buffer)
return &FileChunk{
Data: buffer,
Hash: hash,
Index: index,
OriginalFileMd5: b.FileMd5,
OriginalFileName: fileInfo.Name(),
FileName: fmt.Sprintf("%s.part%d", b.FileMd5+filepath.Ext(fileInfo.Name()), index),
}, nil
}
go-file/binary.go (1.2 KiB)
package gofile
import (
"io"
"os"
)
// ReadEntireFile 读取整个文件到字节切片
func ReadEntireFile(filePath string) ([]byte, error) {
// 使用 os.ReadFile,这是最简单的方法
data, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
return data, nil
}
// ReadFileChunks 分块读取大文件,避免内存占用过大
func ReadFileChunks(filename string, chunkSize int, callback func(chunk []byte) error) error {
file, err := os.Open(filename)
if err != nil {
return err
}
defer file.Close()
// 使用bytes.Buffer来读取文件
for {
// 每次创建新的buffer
chunk := make([]byte, chunkSize)
_, err := file.Read(chunk)
// 调用回调函数处理数据块
// 只传递实际读取的数据
if err := callback(chunk); err != nil {
return err
}
if err == io.EOF {
break
}
if err != nil {
return err
}
}
return nil
}
// ReadFileAt 从指定位置读取文件
func ReadFileAt(filePath string, offset int64, length int) ([]byte, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer file.Close()
data := make([]byte, length)
_, err = file.ReadAt(data, offset)
if err != nil && err != io.EOF {
return nil, err
}
return data, nil
}
go-file/download.go (2.4 KiB)
package gofile
import (
"context"
"fmt"
goerror "github.com/gif-gif/go.io/go-error"
"github.com/zeromicro/go-zero/core/logx"
"net/http"
)
type GoDownload struct {
w http.ResponseWriter
r *http.Request
Logger logx.Logger
ctx context.Context
filename string
}
func NewGoDownload(ctx context.Context, filename string, w http.ResponseWriter, r *http.Request) *GoDownload {
return &GoDownload{
Logger: logx.WithContext(ctx),
ctx: ctx,
w: w,
r: r,
filename: filename,
}
}
func WriteWithBuffer(w http.ResponseWriter, data []byte) error {
// 使用 32KB 的缓冲区
const bufferSize = 4 * 1024
// 分块处理数据
for i := 0; i < len(data); i += bufferSize {
end := i + bufferSize
if end > len(data) {
end = len(data)
}
chunk := data[i:end]
n, err := w.Write(chunk)
if err != nil {
return err
}
if n != len(chunk) {
return fmt.Errorf("short write: %d/%d", n, len(chunk))
}
// 使用 Flush 确保数据发送
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}
return nil
}
func (g *GoDownload) Write(data []byte) error {
return WriteWithBuffer(g.w, data)
}
func (g *GoDownload) WriteString(data string) error {
return g.Write([]byte(data))
}
func (g *GoDownload) Error(err error) {
http.Error(g.w, "UploadError:"+err.Error(), http.StatusInternalServerError)
}
// 二进制读取输出 每次读取4096字节 输出
func (g *GoDownload) Output(filePath string) error {
err := ReadFileChunks(filePath, 4096, func(chunk []byte) error {
err := g.Write(chunk)
return err
})
if err != nil {
g.Error(err)
return err
}
return nil
}
// 一行一行输出
func (g *GoDownload) OutputByLine(filePath string) error {
err := ReadLines(filePath, func(chunk string) error {
err := g.Write([]byte(chunk + "\n"))
return err
})
if err != nil {
g.Error(err)
return err
}
return nil
}
func (g *GoDownload) SetFileHeaders() error {
g.w.Header().Set("Content-Type", "application/octet-stream")
g.w.Header().Set("Content-Disposition", "attachment; filename="+g.filename)
g.w.Header().Set("Cache-Control", "no-cache")
g.w.Header().Set("Connection", "keep-alive")
g.w.WriteHeader(http.StatusOK)
// 使用 Flush 确保数据发送
if f, ok := g.w.(http.Flusher); ok {
f.Flush()
} else {
return goerror.NewErrorMsg(uint32(http.StatusInternalServerError), "Streaming unsupported!")
}
return nil
}
go-file/file.go (3.4 KiB)
package gofile
import (
"bufio"
"fmt"
"io"
"mime/multipart"
"os"
"path"
"path/filepath"
"runtime"
)
const (
EL = "\n"
)
// 文件名
func FILE() string {
_, file, _, _ := runtime.Caller(1)
return file
}
// 行号
func LINE() int {
_, _, line, _ := runtime.Caller(1)
return line
}
// 目录名称
func DIR() string {
_, file, _, _ := runtime.Caller(1)
return path.Dir(file) + "/"
}
// GetFileInfo 获取文件信息
func GetFileInfo(filename string) (size int64, mode os.FileMode, err error) {
info, err := os.Stat(filename)
if err != nil {
return 0, 0, err
}
return info.Size(), info.Mode(), nil
}
// 写文件,支持路径创建
func WriteToFile(filename string, b []byte) error {
dirname := path.Dir(filename)
if _, err := os.Stat(dirname); err != nil {
os.MkdirAll(dirname, 0755)
}
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
if _, err := f.Write(b); err != nil {
return err
}
return nil
}
func Exist(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil // 文件或目录存在
}
if os.IsNotExist(err) {
return false, nil // 文件或目录不存在
}
return false, err // 发生了其他错误,无法确定
}
func CreateSavePath(dst string, perm os.FileMode) error {
err := os.MkdirAll(dst, perm)
if err != nil {
return err
}
return nil
}
func SaveFile(file *multipart.FileHeader, dst string) error {
src, err := file.Open()
if err != nil {
return err
}
defer src.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, src)
return err
}
// ReadEntireFile 读取整个文件到字节切片一样
func GetFileContent(filePath string) ([]byte, error) {
return ReadEntireFile(filePath)
}
func ReadLines(filePath string, lineFunc func(line string) error) error {
// 打开文件
file, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("failed to open file: %v", err)
}
defer file.Close()
// 创建一个新的扫描器
scanner := bufio.NewScanner(file)
// 按行扫描文件
for scanner.Scan() {
line := scanner.Text()
e := lineFunc(line)
if e != nil {
return e
}
}
// 检查扫描过程中是否有错误
if err := scanner.Err(); err != nil {
return fmt.Errorf("error while reading file: %v", err)
}
return nil
}
func GetFileContentString(filePath string) (string, error) {
body, err := GetFileContent(filePath)
if err != nil {
return "", err
}
return string(body), nil
}
// 复制文件
func CopyFile(src, dst string) error {
s, err := os.ReadFile(src)
if err != nil {
return err
}
err = os.WriteFile(dst, s, 0o600)
if err != nil {
return err
}
return nil
}
// 获取当前目录下所有文件
func GetFileList(path string) []string {
var fileList []string
files, _ := os.ReadDir(path)
for _, f := range files {
if !f.IsDir() {
fileList = append(fileList, f.Name())
}
}
return fileList
}
func RemoveFile(file string) error {
return os.Remove(file)
}
func GetDirSize(path string) (int64, error) {
var size int64
// 使用 filepath.Walk 遍历目录
err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// 不是目录就累加文件大小
if !info.IsDir() {
size += info.Size()
}
return nil
})
return size, err
}
func GetFileSize(file string) (int64, error) {
size, _, err := GetFileInfo(file)
return size, err
}
go-file/file_exist.go (168 B)
package gofile
import (
"os"
)
func Exists(filename string) bool {
_, err := os.Stat(filename)
if err == nil || os.IsExist(err) {
return true
}
return false
}
go-file/file_lock.go (522 B)
package gofile
import (
"os"
)
type FileLock struct {
Filename string
fh *os.File
}
func (fl *FileLock) Lock() (err error) {
if fl.Filename == "" {
fl.Filename = ".lock"
}
fl.fh, err = os.Create(fl.Filename)
if err != nil {
return
}
err = Flock(fl.fh.Fd())
return
}
func (fl *FileLock) UnLock() (err error) {
defer fl.release()
if err = Funlock(fl.fh.Fd()); err != nil {
return
}
return
}
func (fl *FileLock) release() {
if fl.fh != nil {
fl.fh.Close()
os.Remove(fl.Filename)
}
}
go-file/file_md5.go (2.0 KiB)
package gofile
import (
"crypto/md5"
"encoding/hex"
"fmt"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"io"
"io/ioutil"
"mime/multipart"
"os"
)
// 计算文件MD5 支持大文件
func MD5(file string) (string, error) {
defaultSize := int64(16 * 1024 * 1024)
info, err := os.Stat(file)
if err != nil {
golog.Error(err)
return "", err
}
// 小文件
if info.Size() < defaultSize {
b, _ := ioutil.ReadFile(file)
sum := md5.Sum(b)
return hex.EncodeToString(sum[:]), nil
}
// 大文件
{
tempFile, err := ioutil.TempFile(os.TempDir(), "goo-md5-temp-file")
if err != nil {
golog.Error(err)
return "", err
}
defer tempFile.Close()
f, err := os.OpenFile(file, os.O_RDONLY, 0755)
if err != nil {
golog.Error(err)
return "", err
}
defer f.Close()
io.Copy(tempFile, f)
tempFile.Seek(0, os.SEEK_SET)
h := md5.New()
io.Copy(h, tempFile)
return hex.EncodeToString(h.Sum(nil)), nil
}
}
// 计算文件md5(支持超大文件)
func CalculateFileMD5(filePath string) (string, error) {
// 打开文件
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()
// 创建MD5哈希对象
hash := md5.New()
// 创建一个缓冲区,逐块读取文件内容
buffer := make([]byte, 1024*1024) // 1MB 缓冲区
for {
n, err := file.Read(buffer)
if err != nil && err != io.EOF {
return "", err
}
if n == 0 {
break
}
// 更新哈希值
if _, err := hash.Write(buffer[:n]); err != nil {
return "", err
}
}
// 计算最终的哈希值
hashInBytes := hash.Sum(nil)
hashInString := fmt.Sprintf("%x", hashInBytes)
return hashInString, nil
}
func GetFileHeaderMd5Name(fileHeader *multipart.FileHeader) (string, error) {
file, err := fileHeader.Open()
if err != nil {
return "", err
}
body, err := io.ReadAll(file)
if err != nil {
return "", err
}
name := goutils.Md5(body)
return name, nil //+ filepath.Ext(fileHeader.Filename)
}
go-file/file_readline.go (596 B)
package gofile
import (
"bufio"
"errors"
golog "github.com/gif-gif/go.io/go-log"
"io"
"os"
)
func ReadByLine(filename string, cb func(b []byte, end bool) error) error {
if !Exists(filename) {
return errors.New("文件不存在")
}
f, err := os.OpenFile(filename, os.O_RDWR, 0755)
if err != nil {
golog.Error(err)
return err
}
defer f.Close()
r := bufio.NewReader(f)
for {
b, err := r.ReadBytes('\n')
if err != nil {
if io.EOF == err {
return cb(b, true)
}
golog.Error(err)
return err
}
if err := cb(b, false); err != nil {
return err
}
}
}
go-file/filelock_unix.go (240 B)
//go:build !windows
// filelock_unix.go
package gofile
import "syscall"
func Flock(fd uintptr) error {
return syscall.Flock(int(fd), syscall.LOCK_EX)
}
func Funlock(fd uintptr) error {
return syscall.Flock(int(fd), syscall.LOCK_UN)
}
go-file/filelock_windows.go (152 B)
//go:build windows
// filelock_windows.go
package gofile
func Flock(fd uintptr) error {
return nil
}
func Funlock(fd uintptr) error {
return nil
}
go-file/readme.md (938 B)
文件操作相关模块
- 文件下载功能
方法 1
func (this FileDownload) DoHandle(ctx *gin.Context) *goserver.Response {
ds := gofile.NewGoDownload(ctx, "downloaded_file.csv", ctx.Writer, ctx.Request)
err := ds.SetFileHeaders()
if err != nil {
return nil
}
filePath := "file.csv"
err = gofile.ReadLines(filePath, func(chunk string) error {
err = ds.Write([]byte(chunk + "\n"))
return err
})
if err != nil {
ds.Error(err)
return nil
}
return nil
}
方法 2
func (this FileDownload) DoHandle(ctx *gin.Context) *goserver.Response {
ds := gofile.NewGoDownload(ctx, "downloaded_file.csv", ctx.Writer, ctx.Request)
err := ds.SetFileHeaders()
file := "file.csv"
err = ds.Output(file)
if err != nil {
http.Error(ctx.Writer, "Streaming unsupported!", http.StatusInternalServerError)
return nil
}
return nil
}
go-file/test/main.go (3.4 KiB)
package main
import (
"errors"
gocontext "github.com/gif-gif/go.io/go-context"
gofile "github.com/gif-gif/go.io/go-file"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gogf/gf/util/gconv"
)
var uploadPath = "/Users/Jerry/Downloads/chrome/fileparts"
var fileName = "test.apk"
func main() {
cutLocalFile()
<-gocontext.WithCancel().Done()
}
func cutHttpFile() {
ts := goutils.MeasureExecutionTime(func() {
filePath := "/Users/Jerry/Downloads/chrome/dy12.9.0.apk"
fileMd5, err := goutils.CalculateFileMD5(filePath)
if err != nil {
golog.WithTag("gofile").Error(err)
}
req := &gofile.BigFile{
File: filePath,
MaxWorkers: 3,
ChunkSize: 1,
FileMd5: fileMd5,
}
req.FileChunkCallback = func(chunk *gofile.FileChunk) error {
golog.WithTag("chunkCount").Info(gconv.String(chunk.Index) + ":" + gconv.String(len(chunk.Data)) + ":" + gconv.String(chunk.Hash))
//存储文件或者上传文件
rst, err := gofile.UploadChunk("http://localhost:20085/bot/api/file-uploader", chunk)
if err != nil {
return err
}
if rst.Code != 0 {
golog.WithTag("gofile").Error(rst.Code)
return errors.New(gconv.String(rst.Code) + ":" + rst.Msg)
}
//time.Sleep(1 * time.Second) // for test
return nil
}
err = req.Start()
if err != nil {
golog.WithTag("gofile").Fatal(err)
}
if req.IsSuccess() {
golog.WithTag("gofile").Info("已处理分片:", len(req.SuccessChunkIndexes))
} else {
golog.WithTag("gofile").Error("分片上传失败")
return
}
rst, err := gofile.MergeChunk("http://localhost:20085/bot/api/file-merge-uploader", &gofile.FileMergeReq{
FileMd5: fileMd5,
TotalChunks: req.ChunkCount,
FileName: fileName,
})
if err != nil {
golog.WithTag("gofile").Error(err.Error())
return
}
if rst.Code != 0 {
golog.WithTag("gofile").Error(rst.Code)
return
}
// 调用合并接口
//_, err = gofile.MergeFileForChunks(uploadPath, fileName, fileMd5, req.ChunkCount)
if err != nil {
golog.WithTag("gofile").Error(err)
}
})
golog.WithTag("cutHttpFile").Info("执行时间:" + gconv.String(ts))
}
func cutLocalFile() {
ts := goutils.MeasureExecutionTime(func() {
filePath := "/Users/Jerry/Downloads/chrome/dy12.9.0.apk"
fileMd5, err := goutils.CalculateFileMD5(filePath)
if err != nil {
golog.WithTag("gofile").Error(err)
}
req := &gofile.BigFile{
File: filePath,
MaxWorkers: 3,
ChunkSize: 1,
FileMd5: fileMd5,
}
req.FileChunkCallback = func(chunk *gofile.FileChunk) error {
golog.WithTag("chunkCount").Info(gconv.String(chunk.Index) + ":" + gconv.String(len(chunk.Data)) + ":" + gconv.String(chunk.Hash))
//存储文件或者上传文件
return gofile.SaveToLocal(uploadPath, chunk)
}
err = req.Start()
if err != nil {
golog.WithTag("gofile").Fatal(err)
}
if req.IsSuccess() {
golog.WithTag("gofile").Info("已处理分片:", len(req.SuccessChunkIndexes))
} else {
golog.WithTag("gofile").Error("分片上传失败")
return
}
// 调用合并接口
rst, err := gofile.MergeFileForChunks(uploadPath, fileName, fileMd5, req.ChunkCount, false)
if err != nil {
golog.WithTag("gofile").Error(err.Error())
return
}
if err != nil {
golog.WithTag("gofile").Error(err)
}
golog.WithTag("gofile").Info(rst)
})
golog.WithTag("cutHttpFile").Info("执行时间:" + gconv.String(ts))
}
go-file/types.go (1014 B)
package gofile
const (
UPLOAD_TYPE_LOCAL = 1
UPLOAD_TYPE_OSS = 2
UPLOAD_TYPE_CHUNK = 3
)
type FileReceiveResult struct {
OriginalFile string //有后缀最终文件路径
FileName string //有后缀最终文件名
FileMd5 string //最终文件Md5
OriginalFileName string //有后缀原始文件名
OriginalShortName string //没有后缀的原始文件名
ChunkCount int64
}
// fmt.Sprintf("%s.part%d", fileName, i)
// mapreduce big file
// 大文件逻辑 for 把大文件并发分片处理,为了防止OOM超大文件边分片边处理的策略
type FileChunk struct {
Data []byte //分片数据
Hash string //分片Hash
Index int64 //分片顺序号
OriginalFileName string //原文件名
OriginalFileMd5 string //原文件Md5
FileName string //分片文件名称
}
type FileMergeReq struct {
FileMd5 string `json:"fileMd5"`
FileName string `json:"fileName"`
TotalChunks int64 `json:"totalChunks"`
}
go-file/upload.go (5.3 KiB)
package gofile
import (
"context"
"fmt"
"github.com/gif-gif/go.io/go-http"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gogf/gf/util/gconv"
"io"
"mime/multipart"
"os"
"path/filepath"
"strings"
)
// 服务器接受file文件到assetsDir目录下,assetsDir 目录不存在则自动创建,返回存储位置
func ReceiveFile(assetsDir string, file *multipart.FileHeader) (*FileReceiveResult, error) {
if ok, _ := Exist(assetsDir); !ok {
err := CreateSavePath(assetsDir, os.ModePerm)
if err != nil {
return nil, err
}
}
fileMd5, err := GetFileHeaderMd5Name(file)
if err != nil {
return nil, err
}
fullName := fileMd5 + filepath.Ext(file.Filename)
fullName = strings.ToLower(fullName)
originalPath := filepath.Join(assetsDir, fullName)
if ok, _ := Exist(originalPath); !ok {
err = SaveFile(file, originalPath)
if err != nil {
return nil, err
}
}
res := &FileReceiveResult{}
res.OriginalFile = originalPath
res.FileName = fullName
res.FileMd5 = fileMd5
lastIndex := strings.LastIndex(file.Filename, ".")
if lastIndex >= 0 {
res.OriginalShortName = file.Filename[:lastIndex]
} else {
res.OriginalShortName = file.Filename
}
res.OriginalFileName = file.Filename
return res, nil
}
// 服务器接受文件分片
func ReceiveChunkHandler(assetsDir string, chunkIndex int64, chunkMd5 string, fileMd5 string, file *multipart.FileHeader) (*FileReceiveResult, error) {
if ok, _ := Exist(assetsDir); !ok {
err := CreateSavePath(assetsDir, os.ModePerm)
if err != nil {
return nil, err
}
}
chunkFileMd5, err := GetFileHeaderMd5Name(file)
if err != nil {
return nil, err
}
if chunkFileMd5 != chunkMd5 { //分片md5 验证
return nil, fmt.Errorf("chunkFileMd5 mismatch")
}
fullName := fileMd5 + filepath.Ext(file.Filename) //以原文件md5作为命名
fullName = strings.ToLower(fullName)
// 创建分片文件
chunkFilePath := filepath.Join(assetsDir, fmt.Sprintf("%s.part%d", fullName, chunkIndex))
if ok, _ := Exist(chunkFilePath); !ok {
err = SaveFile(file, chunkFilePath)
if err != nil {
return nil, err
}
}
res := &FileReceiveResult{}
res.OriginalFile = chunkFilePath
res.FileName = fullName
res.OriginalFileName = file.Filename
return res, nil
}
// 服务器合并所有文件分片,并验证md5, isNotRemoveChunk =true 合并后时不会删除分片
func MergeFileForChunks(filePath string, fileName string, fileMd5 string, totalChunks int64, isNotRemoveChunk bool) (*FileReceiveResult, error) {
fileName = fileMd5 + filepath.Ext(fileName)
finalFilePath := filepath.Join(filePath, fileName)
finalFile, err := os.Create(finalFilePath)
if err != nil {
return nil, err
}
defer finalFile.Close()
// 合并所有分片
for i := 0; i < int(totalChunks); i++ {
chunkFilePath := filepath.Join(filePath, fmt.Sprintf("%s.part%d", fileMd5+filepath.Ext(fileName), i))
chunkFile, err := os.Open(chunkFilePath)
if err != nil {
return nil, err
}
defer chunkFile.Close()
_, err = io.Copy(finalFile, chunkFile)
if err != nil {
return nil, err
}
if !isNotRemoveChunk {
// 删除分片文件
err = os.Remove(chunkFilePath)
if err != nil {
return nil, err
}
}
}
finalFileMd5, err := goutils.CalculateFileMD5(finalFilePath)
if err != nil {
return nil, err
}
if fileMd5 != finalFileMd5 { //最终文件md5验证
return nil, fmt.Errorf("fileMd5 mismatch")
}
res := &FileReceiveResult{
ChunkCount: totalChunks,
OriginalFile: finalFilePath,
FileName: fileName,
}
//golog.WithTag("mergeFile").Info("执行时间:" + gconv.String(ts))
return res, nil
}
// ////////////////////////////////////////////////////////////// http server upload and merge 供参考
// 上传一个文件分片,(作为客户端请求时验证非法请求认证逻辑需要加,如authToken sign 等等)
func UploadChunk(url string, chunk *FileChunk) (*gohttp.Response, error) {
req := &gohttp.Request{
Url: url,
Method: gohttp.POST,
FileBytes: chunk.Data,
MultipartFormData: map[string]string{
"type": gconv.String(UPLOAD_TYPE_CHUNK),
"fileName": chunk.FileName,
"fileMd5": chunk.OriginalFileMd5,
"chunkMd5": chunk.Hash,
"chunkIndex": gconv.String(chunk.Index),
},
FileName: chunk.OriginalFileName,
Headers: map[string]string{"User-Agent": "github.com/gif-gif/go.io"},
}
gh := gohttp.GoHttp[gohttp.Response]{
Request: req,
}
res, err := gh.HttpPost(context.Background())
if err != nil {
return nil, err
}
return res, nil
}
// 分片全部上传完毕后,再请求文件分片合并请求(作为客户端请求时验证非法请求认证逻辑需要加,如authToken sign 等等)
func MergeChunk(url string, fileMergeReq *FileMergeReq) (*gohttp.Response, error) {
req := &gohttp.Request{
Url: url,
Method: gohttp.POST,
Headers: map[string]string{"User-Agent": "github.com/gif-gif/go.io"},
Body: fileMergeReq,
}
gh := gohttp.GoHttp[gohttp.Response]{
Request: req,
}
res, err := gh.HttpPost(context.Background())
if err != nil {
return nil, err
}
return res, nil
}
// //////////////////////////////////////////////////////////////local save
// 把分片存在指定目录
func SaveToLocal(savePath string, chunk *FileChunk) error {
chunkFile := filepath.Join(savePath, chunk.FileName)
err := WriteToFile(chunkFile, chunk.Data)
if err != nil {
return err
}
return nil
}
go-file/zip.go (1.2 KiB)
package gofile
import (
"archive/zip"
"io"
"os"
"path/filepath"
"strings"
)
func isValidFile(name string) bool {
return !strings.HasPrefix(name, "__MACOSX") &&
!strings.HasPrefix(filepath.Base(name), "._")
}
func UnzipFile(zipFile, destDir string) error {
// 打开 zip 文件
reader, err := zip.OpenReader(zipFile)
if err != nil {
return err
}
defer reader.Close()
// 创建目标目录
if err := os.MkdirAll(destDir, 0755); err != nil {
return err
}
// 遍历 zip 文件内容
for _, file := range reader.File {
// 构建完整路径
// 跳过 __MACOSX 目录和 ._ 开头的文件
if !isValidFile(file.Name) {
continue
}
path := filepath.Join(destDir, file.Name)
// 如果是目录,创建它
if file.FileInfo().IsDir() {
os.MkdirAll(path, file.Mode())
continue
}
// 创建目标文件
destFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
if err != nil {
return err
}
// 打开 zip 中的文件
srcFile, err := file.Open()
if err != nil {
destFile.Close()
return err
}
// 复制内容
_, err = io.Copy(destFile, srcFile)
srcFile.Close()
destFile.Close()
if err != nil {
return err
}
}
return nil
}
go-grpc/readme.md (37 B)
Grpc
自定义Grpc框架实现
go-http/gohttpx/const.go (180 B)
package gohttpx
const (
TAG = "gohttp"
CONTENT_TYPE_XML = "application/xml"
CONTENT_TYPE_JSON = "application/json"
CONTENT_TYPE_FORM = "application/x-www-form-urlencoded"
)
go-http/gohttpx/http.go (3.6 KiB)
package gohttpx
import (
goutils "github.com/gif-gif/go.io/go-utils"
"io"
"net/http"
"strings"
)
func GetHeaderIpInfos(r *http.Request, field string) map[string]string {
result := make(map[string]string)
result["X-Forwarded-For"] = r.Header.Get("X-Forwarded-For")
result["X-Forward-For"] = r.Header.Get("X-Forward-For")
result["Proxy-Client-IP"] = r.Header.Get("Proxy-Client-IP")
result["WL-Proxy-Client-IP"] = r.Header.Get("WL-Proxy-Client-IP")
result["HTTP_CLIENT_IP"] = r.Header.Get("HTTP_CLIENT_IP")
result["HTTP_X_FORWARDED_FOR"] = r.Header.Get("HTTP_X_FORWARDED_FOR")
result["X-Real-IP"] = r.Header.Get("X-Real-IP")
result["r-RemoteAddr"] = r.RemoteAddr
return result
}
// GetClientIp returns the client ip of this request without port.
// Note that this ip address might be modified by client header.
func GetClientIp(r *http.Request) string {
clientIp := ""
realIps := r.Header.Get("X-Forwarded-For")
if realIps != "" && len(realIps) != 0 && !strings.EqualFold("unknown", realIps) {
ipArray := strings.Split(realIps, ",")
clientIp = ipArray[0]
if clientIp != "" {
if goutils.IsIPv4(clientIp) {
return clientIp
}
if len(ipArray) > 1 {
clientIp = strings.TrimSpace(ipArray[1])
if goutils.IsIPv4(clientIp) {
return clientIp
}
}
}
}
realIps = r.Header.Get("X-Forward-For")
if realIps != "" && len(realIps) != 0 && !strings.EqualFold("unknown", realIps) {
ipArray := strings.Split(realIps, ",")
clientIp = ipArray[0]
if clientIp != "" {
return clientIp
}
}
if strings.EqualFold("unknown", realIps) {
clientIp = r.Header.Get("Proxy-Client-IP")
}
if clientIp == "" || strings.EqualFold("unknown", realIps) {
clientIp = r.Header.Get("WL-Proxy-Client-IP")
}
if clientIp == "" || strings.EqualFold("unknown", realIps) {
clientIp = r.Header.Get("HTTP_CLIENT_IP")
}
if clientIp == "" || strings.EqualFold("unknown", realIps) {
clientIp = r.Header.Get("HTTP_X_FORWARDED_FOR")
}
if clientIp == "" || strings.EqualFold("unknown", realIps) {
clientIp = r.Header.Get("X-Real-IP")
}
if clientIp == "" || strings.EqualFold("unknown", realIps) {
clientIp = r.RemoteAddr
if idx := strings.LastIndex(clientIp, ":"); idx > 0 {
clientIp = clientIp[:idx]
}
}
return clientIp
}
func New(opts ...Option) *Request {
r := &Request{
Headers: map[string]string{
"Content-Type": CONTENT_TYPE_FORM,
},
}
for _, opt := range opts {
switch opt.Name {
case "tls":
v := opt.Value.(map[string]string)
r.Tls = &Tls{
CaCrtFile: v["caCrtFile"],
ClientCrtFile: v["clientCrtFile"],
ClientKeyFile: v["clientKeyFile"],
}
case "content-type-xml", "content-type-json", "content-type-form":
r.SetHeader("Content-Type", opt.Value.(string))
case "header":
v := opt.Value.(map[string]string)
for field, value := range v {
r.SetHeader(field, value)
}
}
}
return r
}
func Get(url string) ([]byte, error) {
return New().Get(url)
}
func GetWithQuery(url string, data []byte) ([]byte, error) {
return New().GetWithQuery(url, data)
}
func Post(url string, data []byte) ([]byte, error) {
return New().Post(url, data)
}
func PostJson(url string, data []byte) ([]byte, error) {
return New().JsonContentType().Post(url, data)
}
func Put(url string, data []byte) ([]byte, error) {
return New().Put(url, data)
}
func Upload(url, fileField, fileName string, fh io.Reader, data map[string]string) (b []byte, err error) {
return New().Upload(url, fileField, fileName, fh, data)
}
func SetHeader(name, value string) *Request {
return New().SetHeader(name, value)
}
func Debug() *Request {
return New().Debug()
}
go-http/gohttpx/option.go (828 B)
package gohttpx
type Option struct {
Name string
Value interface{}
}
func TlsOption(caCrtFile, clientCrtFile, clientKeyFile string) Option {
return Option{Name: "tls", Value: map[string]string{
"caCrtFile": caCrtFile,
"clientCrtFile": clientCrtFile,
"clientKeyFile": clientKeyFile,
}}
}
func ContentTypeXmlOption() Option {
return Option{Name: "content-type-xml", Value: CONTENT_TYPE_XML}
}
func ContentTypeJsonOption() Option {
return Option{Name: "content-type-xml", Value: CONTENT_TYPE_JSON}
}
func ContentTypeFormOption() Option {
return Option{Name: "content-type-xml", Value: CONTENT_TYPE_FORM}
}
func HeaderOption(field, value string) Option {
return Option{Name: "header", Value: map[string]string{
field: value,
}}
}
func DebugOption() Option {
return Option{Name: "debug", Value: true}
}
go-http/gohttpx/readme.md (427 B)
gohttpx 模块
- 快捷使用http 请求
resp, err := gohttpx.PostJson(Email.Api, data)
if err != nil {
logx.Error("邮件发送失败:" + string(data) + ",error:" + err.Error())
return err
} else {
logx.Infof("executeSendEmail result:%s data:%s", string(resp), string(data))
}
return nil
rst, err := gohttpx.Post(url, []byte(postParams))
clientIp := gohttpx.GetClientIp(request)
go-http/gohttpx/request.go (3.1 KiB)
package gohttpx
import (
"bytes"
"crypto/tls"
"crypto/x509"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"time"
)
type Request struct {
Headers map[string]string
Tls *Tls
timeout time.Duration
debug bool
}
func (r *Request) Debug() *Request {
r.debug = true
return r
}
func (r *Request) SetHeader(name, value string) *Request {
r.Headers[name] = value
return r
}
func (r *Request) SetContentType(contentType string) *Request {
r.SetHeader("Content-Type", contentType)
return r
}
func (r *Request) JsonContentType() *Request {
r.SetHeader("Content-Type", CONTENT_TYPE_JSON)
return r
}
func (r *Request) SetTimeout(d time.Duration) *Request {
r.timeout = d
return r
}
func (r *Request) getClient() *http.Client {
if r.timeout == 0 {
r.timeout = 8 * time.Second
}
client := &http.Client{
Timeout: r.timeout,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
if r.Tls != nil {
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(r.Tls.CaCrt())
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: pool,
Certificates: []tls.Certificate{r.Tls.ClientCrt()},
},
}
}
return client
}
func (r *Request) Do(method, url string, reader io.Reader) (rst []byte, err error) {
var (
req *http.Request
rsp *http.Response
)
req, err = http.NewRequest(method, url, reader)
if err != nil {
return
}
for k, v := range r.Headers {
req.Header.Set(k, v)
}
rsp, err = r.getClient().Do(req)
if err != nil {
return
}
defer rsp.Body.Close()
rst, err = ioutil.ReadAll(rsp.Body)
if err != nil {
return
}
return
}
func (r *Request) handle(method, url string, data []byte) (rsp []byte, err error) {
rsp, err = r.Do(method, url, bytes.NewReader(data))
if r.debug {
l := golog.WithTag(TAG).
WithField("method", method).
WithField("url", url).
WithField("header", r.Headers).
WithField("request-data", string(data)).
WithField("response", string(rsp))
if err != nil {
l.Error()
} else {
l.Debug()
}
}
return
}
func (r *Request) Get(url string) ([]byte, error) {
return r.handle("GET", url, nil)
}
func (r *Request) GetWithQuery(url string, data []byte) ([]byte, error) {
return r.handle("GET", url, data)
}
func (r *Request) Post(url string, data []byte) ([]byte, error) {
return r.handle("POST", url, data)
}
func (r *Request) PostJson(url string, data []byte) ([]byte, error) {
return r.JsonContentType().handle("POST", url, data)
}
func (r *Request) Put(url string, data []byte) ([]byte, error) {
return r.handle("PUT", url, data)
}
func (r *Request) Upload(url, fileField, fileName string, fh io.Reader, data map[string]string) ([]byte, error) {
pr, pw := io.Pipe()
w := multipart.NewWriter(pw)
goutils.AsyncFunc(func() {
for k, v := range data {
w.WriteField(k, v)
}
part, _ := w.CreateFormFile(fileField, fileName)
io.CopyBuffer(part, fh, nil)
w.Close()
pw.Close()
})
r.SetHeader("Content-Type", w.FormDataContentType())
return r.Do("POST", url, pr)
}
go-http/gohttpx/tls.go (534 B)
package gohttpx
import (
"crypto/tls"
golog "github.com/gif-gif/go.io/go-log"
"io/ioutil"
)
type Tls struct {
CaCrtFile string
ClientCrtFile string
ClientKeyFile string
}
func (s *Tls) CaCrt() []byte {
if s.CaCrtFile == "" {
return caCert
}
bts, err := ioutil.ReadFile(s.CaCrtFile)
if err != nil {
golog.Error(err.Error())
}
return bts
}
func (s *Tls) ClientCrt() tls.Certificate {
crt, err := tls.LoadX509KeyPair(s.ClientCrtFile, s.ClientKeyFile)
if err != nil {
golog.Error(err.Error())
}
return crt
}
go-http/gowebsocket/client.go (6.8 KiB)
package gowebsocket
import (
"fmt"
"math/rand"
"net/url"
"os"
"os/signal"
"sync"
"time"
"github.com/gogf/gf/util/gconv"
"github.com/gorilla/websocket"
"github.com/zeromicro/go-zero/core/logx"
)
type ClientConfig struct {
Name string `yaml:"Name" json:"name,optional"`
Port int `yaml:"Port" json:"port,optional"`
Addr string `yaml:"Addr" json:"addr,optional"`
ClientID string `yaml:"ClientID" json:"clientID,optional"`
HeartBeatInterval int64 `yaml:"HeartBeatInterval" json:"heartBeatInterval,optional"`
}
// WebSocket 客户端
type WSClient struct {
conn *websocket.Conn
clientID string
heartbeatInterval time.Duration
lastHeartbeat time.Time
isConnected bool
mutex sync.RWMutex
send chan Message // 发送通道
done chan struct{}
interrupt chan os.Signal
ClientConfig ClientConfig
}
// 创建新的 WebSocket 客户端
func NewWSClient(config ClientConfig) *WSClient {
c := &config
if c.HeartBeatInterval == 0 {
c.HeartBeatInterval = 20
}
return &WSClient{
ClientConfig: config,
clientID: config.ClientID,
heartbeatInterval: time.Duration(config.HeartBeatInterval) * time.Second, // 每20秒发送一次心跳
isConnected: false,
done: make(chan struct{}),
send: make(chan Message, 256),
interrupt: make(chan os.Signal, 1),
}
}
// 连接到服务器
func (c *WSClient) Connect() error {
u := url.URL{Scheme: "ws", Host: c.ClientConfig.Addr + ":" + gconv.String(c.ClientConfig.Port), Path: "/ws"}
logx.Infof("[%s] 正在连接到 %s", c.clientID, u.String())
conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
if err != nil {
return fmt.Errorf("连接失败: %v", err)
}
c.mutex.Lock()
c.conn = conn
c.isConnected = true
c.lastHeartbeat = time.Now()
c.mutex.Unlock()
logx.Infof("[%s] 连接成功", c.clientID)
// 设置信号处理
signal.Notify(c.interrupt, os.Interrupt)
return nil
}
// 启动客户端
func (c *WSClient) Start() {
// 启动各个协程
go c.readPump()
go c.writePump()
go c.chatPump() //for test
// 等待中断信号或连接断开
select {
case <-c.done:
logx.Infof("[%s] 连接已断开", c.clientID)
case <-c.interrupt:
logx.Infof("[%s] 收到中断信号,正在关闭连接...", c.clientID)
}
c.Close()
}
// 读取消息协程
func (c *WSClient) readPump() {
defer func() {
c.Close()
}()
for {
var msg Message
err := c.conn.ReadJSON(&msg)
if err != nil {
if websocket.IsUnexpectedCloseError(err,
websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
logx.Infof("[%s] 读取消息错误: %v", c.clientID, err)
}
break
}
c.handleMessage(msg)
}
}
// 写入协程 - 唯一的写入点
func (c *WSClient) writePump() {
// 心跳定时器
ticker := time.NewTicker(c.heartbeatInterval)
defer func() {
ticker.Stop()
c.Close()
logx.Infof("客户端 %s 写入协程退出", c.clientID)
}()
for {
select {
case message, ok := <-c.send:
// 设置写入超时
c.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
if !ok {
// 发送通道已关闭,发送关闭消息
logx.Infof("客户端 %s 发送通道关闭", c.clientID)
c.conn.WriteMessage(websocket.CloseMessage, []byte{})
return
}
// 写入消息
if err := c.conn.WriteJSON(message); err != nil {
logx.Infof("客户端 %s 写入消息失败: %v", c.clientID, err)
return
}
logx.Infof("客户端 %s 发送消息: [%s] %s", c.clientID, message.Type, message.Content)
case <-ticker.C:
// 发送心跳 Ping
heartbeat := Message{
Type: MsgTypeHeartbeat,
Content: "ping",
Timestamp: time.Now(),
}
if !c.sendMessage(heartbeat) {
logx.Infof("[%s] 发送心跳失败: %v", c.clientID)
return
}
c.mutex.Lock()
c.lastHeartbeat = time.Now()
c.mutex.Unlock()
logx.Infof("[%s] 发送心跳", c.clientID)
case <-c.done:
logx.Infof("客户端 %s 收到关闭信号", c.clientID)
return
}
}
}
// 模拟聊天消息发送协程
func (c *WSClient) chatPump() {
ticker := time.NewTicker(45 * time.Second) // 每45秒发送一条聊天消息
defer ticker.Stop()
messages := []string{
"Hello from client!",
"How is everyone doing?",
"This is a test message",
"WebSocket is working great!",
"Hope everyone is having a good day",
}
for {
select {
case <-ticker.C:
if !c.IsConnected() {
return
}
// 随机选择一条消息
content := messages[rand.Intn(len(messages))]
chatMsg := Message{
Type: MsgTypeChat,
Content: fmt.Sprintf("[%s] %s", c.clientID, content),
Timestamp: time.Now(),
}
if !c.sendMessage(chatMsg) {
logx.Infof("[%s] 发送聊天消息失败: %v", c.clientID)
return
}
logx.Infof("[%s] 发送聊天消息: %s", c.clientID, content)
case <-c.done:
return
}
}
}
// 处理接收到的消息
func (c *WSClient) handleMessage(msg Message) {
switch msg.Type {
case MsgTypeHeartbeatAck:
logx.Infof("[%s] 收到心跳确认", c.clientID)
case MsgTypeSystem:
logx.Infof("[%s] 系统消息: %s", c.clientID, msg.Content)
case MsgTypeChat:
if msg.ClientID != c.clientID { // 不显示自己的消息
logx.Infof("[%s] 聊天消息: %s", c.clientID, msg.Content)
}
default:
logx.Infof("[%s] 未知消息类型 [%s]: %s", c.clientID, msg.Type, msg.Content)
}
}
// 发送消息
func (c *WSClient) sendMessage(msg Message) bool {
//c.mutex.RLock()
//defer c.mutex.RUnlock()
//c.mutex.Lock()
//defer c.mutex.Unlock()
//if !c.isConnected || c.conn == nil {
// return fmt.Errorf("连接已断开")
//}
//
//c.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
//return c.conn.WriteJSON(msg)
c.mutex.Lock()
defer c.mutex.Unlock()
if !c.isConnected {
logx.Infof("客户端 %s 已关闭,无法发送消息", c.clientID)
return false
}
select {
case c.send <- msg:
return true
case <-time.After(5 * time.Second): // 防止阻塞
logx.Infof("客户端 %s 发送消息超时", c.clientID)
return false
}
}
// 检查连接状态
func (c *WSClient) IsConnected() bool {
c.mutex.RLock()
defer c.mutex.RUnlock()
return c.isConnected
}
// 关闭连接
func (c *WSClient) Close() {
c.mutex.Lock()
defer c.mutex.Unlock()
if c.isConnected {
c.isConnected = false
// 发送关闭消息
if c.conn != nil {
c.conn.WriteMessage(websocket.CloseMessage,
websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
c.conn.Close()
}
// 通知所有协程退出
select {
case <-c.done:
// 已经关闭
default:
close(c.done)
}
logx.Infof("[%s] 连接已关闭", c.clientID)
}
}
// 获取最后心跳时间
func (c *WSClient) GetLastHeartbeat() time.Time {
c.mutex.RLock()
defer c.mutex.RUnlock()
return c.lastHeartbeat
}
go-http/gowebsocket/readme.md (1.4 KiB)
GoWebsocket
Server
// server.go
package main
import (
"log"
gocontext "github.com/gif-gif/go.io/go-context"
"github.com/gif-gif/go.io/go-http/gowebsocket"
)
func main() {
gowebsocket.InitServer(gowebsocket.ServerConfig{
Port: 8080,
})
if err := gowebsocket.DefaultServer().Start(); err != nil {
log.Fatalf("服务器启动失败: %v", err)
}
<-gocontext.Cancel().Done()
}
Client
// client.go
package main
import (
"flag"
"fmt"
"log"
"math/rand"
gocontext "github.com/gif-gif/go.io/go-context"
"github.com/gif-gif/go.io/go-http/gowebsocket"
)
func main() {
var addr = flag.String("addr", "localhost", "服务器地址")
var clientID = flag.String("id", "", "客户端ID")
flag.Parse()
// 如果没有指定客户端ID,则自动生成
if *clientID == "" {
*clientID = fmt.Sprintf("client_%d", rand.Int63())
}
gowebsocket.InitClient(gowebsocket.ClientConfig{
ClientID: *clientID,
Addr: *addr,
Port: 8080,
HeartBeatInterval: 10,
})
// 连接到服务器
if err := gowebsocket.DefaultClient().Connect(); err != nil {
log.Fatalf("连接失败: %v", err)
}
log.Printf("[%s] 客户端启动", *clientID)
log.Printf("[%s] 心跳间隔: %v", *clientID, gowebsocket.DefaultClient().ClientConfig.HeartBeatInterval)
// 启动客户端
gowebsocket.DefaultClient().Start()
log.Printf("[%s] 客户端已退出", *clientID)
<-gocontext.Cancel().Done()
}
go-http/gowebsocket/types.go (488 B)
package gowebsocket
import (
"time"
)
// 消息类型常量
// 消息类型常量
const (
MsgTypeHeartbeat = "heartbeat"
MsgTypeHeartbeatAck = "heartbeat_ack"
MsgTypeChat = "chat"
MsgTypeSystem = "system"
MsgTypeShutdown = "shutdown"
)
// 消息结构体
type Message struct {
Type string `json:"type"`
Content string `json:"content"`
ClientID string `json:"client_id"`
Timestamp time.Time `json:"timestamp"`
}
// 客户端连接结构
go-http/gowebsocket/websocket.go (2.5 KiB)
package gowebsocket
import (
"errors"
"sync"
"time"
golog "github.com/gif-gif/go.io/go-log"
)
var mu sync.RWMutex
var __servers = map[string]*Server{}
var __clients = map[string]*WSClient{}
// server--------------
// 可以一次初始化多个Redis实例或者 多次调用初始化多个实例
func InitServer(configs ...ServerConfig) (err error) {
for _, conf := range configs {
mu.Lock()
name := conf.Name
if name == "" {
name = "default"
}
if __servers[name] != nil {
return errors.New("Hub server [" + name + "] already exists")
}
__servers[name] = NewServer(conf)
mu.Unlock()
}
return nil
}
func GetServer(names ...string) *Server {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __servers[name]; ok {
return cli
}
return nil
}
func DelServer(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
r := GetServer(name)
if r != nil {
if r.hub != nil {
err := r.GracefulShutdown(30 * time.Second)
if err != nil {
golog.WithTag("GracefulShutdown").Error("err:" + err.Error())
}
}
}
delete(__servers, name)
}
}
}
func DefaultServer() *Server {
if cli, ok := __servers["default"]; ok {
return cli
}
if l := len(__servers); l == 1 {
for _, cli := range __servers {
return cli
}
}
golog.WithTag("Hub").Error("no default websocket server")
return nil
}
// client ------------
// 可以一次初始化多个Redis实例或者 多次调用初始化多个实例
func InitClient(configs ...ClientConfig) (err error) {
for _, conf := range configs {
mu.Lock()
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("Hub client [" + name + "] already exists")
}
__clients[name] = NewWSClient(conf)
mu.Unlock()
}
return nil
}
func GetClient(names ...string) *WSClient {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
r := GetClient(name)
if r != nil {
r.Close()
}
delete(__clients, name)
}
}
}
func DefaultClient() *WSClient {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("Hub").Error("no default websocket client")
return nil
}
go-http/http.go (5.7 KiB)
package gohttp
import (
"bytes"
"context"
"errors"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/go-resty/resty/v2"
"github.com/gogf/gf/util/gconv"
"net/http"
"strings"
"sync"
"time"
)
type GoHttp[T any] struct {
BaseUrl string
Headers map[string]string
Request *Request
}
func (g *GoHttp[T]) SetBaseUrl(base string) {
g.BaseUrl = base
}
func (g *GoHttp[T]) GetBaseUrl() string {
return g.BaseUrl
}
func (g *GoHttp[T]) AddGlobalHeader(k, v string) {
if g.Headers == nil {
g.Headers = make(map[string]string)
}
g.Headers[k] = v
}
func (g *GoHttp[T]) RemoveGlobalHeader(k string) {
delete(g.Headers, k)
}
func (g *GoHttp[T]) GetGlobalHeaders() map[string]string {
return g.Headers
}
// Headers["Accept"] = "application/json" for default
// 真正的请求逻辑
func (g *GoHttp[T]) doHttpRequest(context context.Context, req *Request) (*T, error) {
if req.Url == "" || !strings.HasPrefix(req.Url, "http") {
req.Url = g.GetBaseUrl() + req.Url
}
if req.Url == "" || !strings.HasPrefix(req.Url, "http") {
return nil, errors.New("[" + gconv.String(HttpParamsError) + "]" + "url is invalid")
}
if req.Timeout <= 0 {
req.Timeout = time.Second * 10
}
var (
restyClient = resty.New().
SetTimeout(req.Timeout).
SetRetryCount(req.RetryCount).
SetRetryWaitTime(req.RetryWaitTime)
)
//if 2 {
// restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
//}
//
//if 1 {
// restyClient.SetCertificates(certFile, keyFile, password)
//}
if req.proxyURL != "" {
restyClient.SetProxy(req.proxyURL)
}
if req.Headers == nil {
req.Headers = make(map[string]string)
req.Headers["Accept"] = CONTENT_TYPE_JSON
} else {
req.Headers["Accept"] = CONTENT_TYPE_JSON
}
for k, v := range g.GetGlobalHeaders() {
req.Headers[k] = v
}
var t T
var resp *resty.Response
var err error
request := restyClient.R()
request.SetContext(context)
if !req.BinaryResponse {
request.SetResult(t)
}
request.SetHeaders(req.Headers)
if req.QueryParams != nil {
request.SetQueryParams(req.QueryParams)
}
if req.ParamsValues != nil {
request.SetQueryParamsFromValues(req.ParamsValues)
}
if req.Body != nil {
request.SetBody(req.Body)
}
if req.Method == POST {
if len(req.FormData) > 0 {
request.SetFormData(req.FormData)
}
if len(req.MultipartFormData) > 0 {
request.SetMultipartFormData(req.MultipartFormData)
}
if len(req.FileBytes) > 0 {
request.SetFileReader("file", req.FileName, bytes.NewReader(req.FileBytes))
}
if len(req.Files) > 0 {
request.SetFiles(req.Files)
}
resp, err = request.Post(req.Url)
} else if req.Method == GET {
resp, err = request.Get(req.Url)
} else if req.Method == PUT {
resp, err = request.Put(req.Url)
} else {
resp, err = request.Delete(req.Url)
}
if err != nil {
return nil, err
}
if resp.StatusCode() != http.StatusOK {
return nil, errors.New("[" + gconv.String(resp.StatusCode()) + "]" + "http request error->" + string(resp.Body()))
}
req.TraceInfo = resp.Request.TraceInfo() //调试信息
req.ResponseProto = resp.Proto()
req.ResponseTime = resp.Time()
req.Response = resp
if !req.BinaryResponse { //返回结果如果是二进制直接返回,否则解析泛化处理
respData, ok := resp.Result().(*T)
if !ok {
return nil, errors.New("[" + gconv.String(resp.StatusCode()) + "]" + "Response T is invalid")
}
if respData == nil {
return nil, errors.New("[" + gconv.String(resp.StatusCode()) + "]" + "Response data is empty")
}
return respData, nil
}
return nil, nil
}
func (g *GoHttp[T]) HttpPostJson(context context.Context) (*T, error) {
if g.Request.Headers == nil {
g.Request.Headers = make(map[string]string)
}
g.Request.Headers["Content-Type"] = CONTENT_TYPE_JSON
g.Request.Method = POST
return g.HttpRequest(context)
}
func (g *GoHttp[T]) HttpPost(context context.Context) (*T, error) {
g.Request.Method = POST
return g.HttpRequest(context)
}
func (g *GoHttp[T]) HttpGet(context context.Context) (*T, error) {
g.Request.Method = GET
return g.HttpRequest(context)
}
// 带多个Urls重试逻辑
func (g *GoHttp[T]) HttpRequest(context context.Context) (*T, error) {
t, err := g.doHttpRequest(context, g.Request)
if err == nil {
return t, nil
} else {
if len(g.Request.Urls) == 0 { //没有重试urls
return t, err
}
errs := errors.New("HttpRetryError error")
errs = errors.Join(errs, err)
for _, url := range g.Request.Urls {
g.Request.Url = url
t, err = g.doHttpRequest(context, g.Request)
if err == nil { //请求成功了直接返回
return t, nil
} else {
errs = errors.Join(errs, err)
}
}
return nil, errs // 所有连接重试失败
}
}
// 带多个Urls重试逻辑,并发请求,速度快先到达后 直接返回,其他请求取消
func (g *GoHttp[T]) HttpConcurrencyRequest() (*T, error) {
if g.Request.Url != "" { //把当前加进来起并发
g.Request.Urls = append(g.Request.Urls, g.Request.Url)
}
if len(g.Request.Urls) == 0 { //没有urls
return nil, errors.New("urls is empty")
}
var rst *T
errs := errors.New("Concurrency error")
var one sync.Once
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
fns := []func(){}
for _, url := range g.Request.Urls {
reqNew := *g.Request
reqNew.Url = url
fns = append(fns, func() {
if goutils.IsContextDone(ctx) {
return
}
t, err := g.doHttpRequest(ctx, &reqNew)
if err != nil {
errs = errors.Join(errs, err)
} else { //请求成功了应该直接返回,剩下的请求结果忽略
one.Do(func() { //只保留最快成功的
rst = t
})
cancel() //有一个成功的取消所有请求
}
})
}
goutils.AsyncFuncGroup(fns...)
if goutils.IsContextDone(ctx) { //等待最快的返回后取消其他的上下文信号
return rst, nil
}
return nil, errs
}
go-http/readme.md (1.0 KiB)
Http 请求封装
package main
import (
"context"
"fmt"
gocontext "github.com/gif-gif/go.io/go-context"
"github.com/gif-gif/go.io/go-http"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"time"
)
func main() {
goio.Init(goio.DEVELOPMENT)
type httpRequest struct {
Email string `json:"email"`
}
req := &gohttp.Request{
Url: "/main",
Urls: []string{
"/main1",
"/main2",
"/main3",
},
QueryParams: map[string]string{"name": "jk"},
Timeout: time.Second * 2,
Body: &httpRequest{
Email: "[email protected]",
},
}
gh := &gohttp.GoHttp[gohttp.Response]{
Request: req,
BaseUrl: "http://localhost",
Headers: map[string]string{
"User-Agent": "github.com/gif-gif/go.io",
},
}
rst, err := gh.HttpPostJson(context.Background())
if err != nil {
golog.WithTag("http").Error(err.Error())
} else {
fmt.Println(rst)
}
}
- Custom Root Certificates and Client Certificates
- Custom Root Certificates and Client Certificates from string
- Save HTTP Response into File
go-http/test/main.go (1.6 KiB)
package main
import (
"context"
"fmt"
gocontext "github.com/gif-gif/go.io/go-context"
"github.com/gif-gif/go.io/go-http"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"github.com/zeromicro/go-zero/core/logx"
"time"
)
func main() {
goio.Init(goio.DEVELOPMENT)
logx.DisableStat()
testRaceSpeed()
<-gocontext.WithCancel().Done()
}
func testRequest() {
type httpRequest struct {
Email string `json:"email"`
}
req := &gohttp.Request{
Url: "/main",
Urls: []string{
"/main1",
"/main2",
"/main3",
},
QueryParams: map[string]string{"name": "jk"},
Timeout: time.Second * 2,
Body: &httpRequest{
Email: "[email protected]",
},
}
gh := &gohttp.GoHttp[gohttp.Response]{
Request: req,
BaseUrl: "http://localhost",
Headers: map[string]string{
"User-Agent": "github.com/gif-gif/go.io",
},
}
rst, err := gh.HttpPostJson(context.Background())
if err != nil {
golog.WithTag("http").Error(err.Error())
} else {
fmt.Println(rst)
}
}
func testRaceSpeed() {
type httpRequest struct {
Email string `json:"email"`
}
req := gohttp.Request{
Method: gohttp.POST,
Urls: []string{
"https://jumpjump.io/api/jump/account/check",
"http://localhost:400",
},
QueryParams: map[string]string{"name": "jk"},
Timeout: time.Second * 2,
}
req.Body = &httpRequest{
Email: "[email protected]",
}
gh := &gohttp.GoHttp[gohttp.Response]{
Request: &req,
}
rst, err := gh.HttpConcurrencyRequest()
if err != nil {
golog.ErrorF("Error: \n", err.Error())
} else {
golog.InfoF("res: \n", rst.Code, rst.Msg)
}
}
go-http/types.go (3.9 KiB)
package gohttp
import (
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/go-resty/resty/v2"
"net/url"
"time"
)
const (
TAG = "gohttp"
CONTENT_TYPE_XML = "application/xml"
CONTENT_TYPE_JSON = "application/json"
CONTENT_TYPE_FORM = "application/x-www-form-urlencoded"
)
const (
POST = "post"
GET = "get"
PUT = "put"
DELETE = "delete"
)
const (
HttpUnknownError = 1000
HttpRetryError = 2000
HttpParamsError = 3000
)
type Request struct {
Url string
Urls []string // 如果有值,当url 请求失败时继续用这里的接口尝试,直到成功返回或者全部失败
Method string
Body interface{} //post body 参数
QueryParams map[string]string //get 参数
ParamsValues url.Values //get 参数
FormData map[string]string //formdata 参数
Headers map[string]string
Files map[string]string //上传文件列表
FileName string //文件名称
MultipartFormData map[string]string
FileBytes []byte
Timeout time.Duration
RetryCount int
RetryWaitTime time.Duration
proxyURL string
BinaryResponse bool //是否二进制形式返回,true时不会解析body,直接返回body,不会用T 解析
SetCloseConnection bool //是否关闭连接
//gotrace infos
TraceInfo resty.TraceInfo
ResponseTime time.Duration
ResponseProto string
Response *resty.Response
}
//type HttpError struct {
// HttpStatusCode int
// Msg string
// Error error
// Errors []*HttpError //重试逻辑的错误列表
//}
type res struct{}
type Response struct {
Code int64 `json:"code"`
Msg string `json:"msg"`
Data interface{} `json:"data"`
}
func (r *Request) SetMethod(method string) {
r.Method = method
}
func (r *Request) SetBody(body interface{}) {
r.Body = body
}
func (r *Request) SetTimeout(timeout time.Duration) {
r.Timeout = timeout
}
func (r *Request) SetRetryCount(tryCount int) {
r.RetryCount = tryCount
}
func (r *Request) SetRetryWaitTime(waitTime time.Duration) {
r.RetryWaitTime = waitTime
}
func (r *Request) SetHeader(name string, value string) {
if r.Headers == nil {
r.Headers = make(map[string]string)
}
r.Headers[name] = value
}
func (r *Request) SetQueryParams(name string, value string) {
if r.QueryParams == nil {
r.QueryParams = make(map[string]string)
}
r.QueryParams[name] = value
}
func (r *Request) setUrl(url string) {
r.Url = url
}
// 重复会去重
func (r *Request) AddUrl(url string) {
if !goutils.IsInArray[string](r.Urls, url) {
r.Urls = append(r.Urls, url)
}
}
// 请求成功
func HttpSuccess(data interface{}) *Response {
return &Response{
Code: 0,
Msg: "ok",
Data: data,
}
}
func HttpSuccessByCode(code int64, data interface{}) *Response {
return &Response{
Code: code,
Msg: "ok",
Data: data,
}
}
// 请求格式错误,比如参数格式、参数字段名等 不正确
func HttpBadRequest(msg string) *Response {
return &Response{
Code: 400,
Msg: msg,
Data: res{},
}
}
// 用户没有访问权限,需要进行身份认证
func HttpUnauthorized(msg string) *Response {
return &Response{
Code: 401,
Msg: msg,
Data: res{},
}
}
// 用户已进行身份认证,但权限不够
func HttpForbidden(msg string) *Response {
return &Response{
Code: 403,
Msg: msg,
Data: res{},
}
}
// 接口不存在
func HttpNotFound(msg string) *Response {
return &Response{
Code: 404,
Msg: msg,
Data: res{},
}
}
// 服务器内部错误
func HttpServerError(msg string) *Response {
return &Response{
Code: 500,
Msg: msg,
Data: res{},
}
}
// 请求失败
func HttpFail(msg string) *Response {
return &Response{
Code: 10001,
Msg: msg,
Data: res{},
}
}
// 如需返回特殊错误码,调用此接口
func HttpFailForCode(code int64, msg string) *Response {
return &Response{
Code: code,
Msg: msg,
Data: res{},
}
}
go-http/utils.go (2.5 KiB)
package gohttp
import (
"context"
"github.com/gif-gif/go.io/go-utils/gocrypto"
"github.com/gif-gif/go.io/go-utils/gozip"
goserver "github.com/gif-gif/go.io/goio/server"
)
// POST 压缩请求
func CompressRequest(url string, body []byte, compressMethod string, compressType string, headers map[string]string) ([]byte, error) {
_, data, err := gozip.Compress(body, compressMethod, compressType)
if err != nil {
return nil, err
}
payload := &Request{
BinaryResponse: true,
Url: url,
Method: POST,
Headers: headers,
Body: data,
}
gh := GoHttp[goserver.Response]{
Request: payload,
}
_, err = gh.HttpPost(context.Background())
if err != nil {
return nil, err
}
compressMethod = payload.Response.Header().Get(gozip.XNlContentEncoding)
_, resData, err := gozip.Compress(payload.Response.Body(), compressMethod, gozip.UnGoZipType)
if err != nil {
return nil, err
}
return resData, nil
}
// EncryptRequest 加密请求 aes cbc
func EncryptRequest(url string, body []byte, reqAesKey []byte, resAesKey []byte, compressMethod string, headers map[string]string) ([]byte, error) {
data, err := gocrypto.GoDataEncrypt(body, reqAesKey, compressMethod)
if err != nil {
return nil, err
}
payload := &Request{
BinaryResponse: true,
Url: url,
Method: POST,
Headers: headers,
Body: data,
}
gh := GoHttp[goserver.Response]{
Request: payload,
}
_, err = gh.HttpPost(context.Background())
if err != nil {
return nil, err
}
compressMethod = payload.Response.Header().Get("X-NL-Content-Encoding")
resData, err := gocrypto.GoDataDecrypt(payload.Response.Body(), resAesKey, compressMethod)
if err != nil {
return nil, err
}
return resData, nil
}
// aes ctr 加密请求
func EncryptCTRRequest(url string, body []byte, aesKey []byte, aesIv []byte, compressMethod string, headers map[string]string) ([]byte, error) {
data, err := gocrypto.GoDataAesCTRTransformEncode(body, aesKey, aesIv, compressMethod)
if err != nil {
return nil, err
}
payload := &Request{
BinaryResponse: true,
Url: url,
Method: POST,
Headers: headers,
Body: data,
}
gh := GoHttp[goserver.Response]{
Request: payload,
}
_, err = gh.HttpPost(context.Background())
if err != nil {
return nil, err
}
compressMethod = payload.Response.Header().Get("X-NL-Content-Encoding")
resData, err := gocrypto.GoDataAesCTRTransformDecode(payload.Response.Body(), aesKey, aesIv, compressMethod)
if err != nil {
return nil, err
}
return resData, nil
}
go-ip/README.md (1.2 KiB)
go-ip
查询IP对应的国家 运营商等信息
https://www.ip2location.com/development-libraries/ip2location/go
package main
import (
"context"
goip "github.com/gif-gif/go.io/go-ip"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"time"
)
func main() {
goio.Init(goio.DEVELOPMENT)
config := goip.Config{
//Mmdb: "GeoLite2-Country.mmdb",
//Ip2locationDB: "IP-COUNTRY-REGION-CITY-ISP.BIN",
IpServiceUrl: "",
}
err := goip.Init(config)
if err != nil {
golog.WithTag("goip").Error(err.Error())
return
}
ipinfo, err := goip.Default().GetIpLocation(context.Background(), "172.99.189.235")
if err != nil {
golog.WithTag("goip").Error(err.Error())
return
}
golog.WithTag("goip").Info(ipinfo)
ipZhInfo, err := goip.Default().QueryDbReaderCountryForZhName("172.99.189.235")
if err != nil {
golog.WithTag("goip").Error(err.Error())
return
}
golog.WithTag("goip").Info(ipZhInfo)
ipInfo, err := goip.Default().QueryLocationInfoByIp("172.99.189.235")
if err != nil {
golog.WithTag("goip").Error(err.Error())
return
}
golog.WithTag("goip").Info(ipInfo)
time.Sleep(time.Second * 5)
}
go-ip/cidr.go (2.1 KiB)
package goip
import (
"encoding/binary"
"fmt"
"net"
)
type CIDR struct {
Network string
FirstIP string
LastIP string
Total uint64
Netmask string
Wildcard string
IPVersion string
}
// PrintIPRange 打印指定 CIDR 范围内的所有 IP
func PrintIPRange(cidrStr string) ([]string, error) {
_, ipnet, err := net.ParseCIDR(cidrStr)
if err != nil {
return nil, fmt.Errorf("invalid CIDR: %v", err)
}
// 获取起始 IP
ip := ipnet.IP
// 将 IP 转换为 4 字节表示(针对 IPv4)
ipStart := binary.BigEndian.Uint32(ip.To4())
// 计算掩码
mask := binary.BigEndian.Uint32(ipnet.Mask)
// 计算最后一个 IP
ipEnd := (ipStart & mask) | (^mask)
ips := []string{}
// 打印范围内的所有 IP
for i := ipStart; i <= ipEnd; i++ {
// 转换回 IP 地址格式
ip := make(net.IP, 4)
binary.BigEndian.PutUint32(ip, i)
fmt.Println(ip)
ips = append(ips, ip.String())
}
return ips, nil
}
func CalculateCIDR(cidrStr string) (*CIDR, error) {
_, ipnet, err := net.ParseCIDR(cidrStr)
if err != nil {
return nil, fmt.Errorf("invalid CIDR format: %v", err)
}
cidr := &CIDR{
Network: cidrStr,
}
if len(ipnet.IP) == net.IPv6len {
cidr.IPVersion = "IPv6"
} else {
cidr.IPVersion = "IPv4"
}
cidr.FirstIP = ipnet.IP.String()
mask := net.IP(ipnet.Mask)
lastIP := make(net.IP, len(ipnet.IP))
for i := 0; i < len(ipnet.IP); i++ {
lastIP[i] = ipnet.IP[i] | ^mask[i]
}
cidr.LastIP = lastIP.String()
ones, bits := ipnet.Mask.Size()
cidr.Total = 1 << uint64(bits-ones)
cidr.Netmask = net.IP(ipnet.Mask).String()
wildcard := make(net.IP, len(ipnet.Mask))
for i := 0; i < len(ipnet.Mask); i++ {
wildcard[i] = ^ipnet.Mask[i]
}
cidr.Wildcard = net.IP(wildcard).String()
return cidr, nil
}
func IsIPInCIDR(ipStr, cidrStr string) (bool, error) {
// 解析 IP 地址
ip := net.ParseIP(ipStr)
if ip == nil {
return false, fmt.Errorf("invalid IP address: %s", ipStr)
}
// 解析 CIDR
_, ipnet, err := net.ParseCIDR(cidrStr)
if err != nil {
return false, fmt.Errorf("invalid CIDR: %s, error: %v", cidrStr, err)
}
// 检查 IP 是否在范围内
return ipnet.Contains(ip), nil
}
go-ip/client.go (991 B)
package goip
import (
"errors"
)
var __clients = map[string]*GoIp{}
// 可以一次初始化多个Redis实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name], err = New(conf)
if err != nil {
return
}
}
return
}
func GetClient(names ...string) *GoIp {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoIp {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
return nil
}
go-ip/config.go (299 B)
package goip
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
Mmdb string `yaml:"Mmdb" json:"mmdb,optional"`
Ip2locationDB string `yaml:"Ip2locationDB" json:"ip2locationDB,optional"`
IpServiceUrl string `yaml:"IpServiceUrl" json:"ipServiceUrl,optional"`
}
go-ip/goip.go (4.7 KiB)
package goip
import (
"context"
"fmt"
gocache "github.com/gif-gif/go.io/go-cache"
gohttp "github.com/gif-gif/go.io/go-http"
"github.com/ip2location/ip2location-go/v9"
"github.com/oschwald/geoip2-golang"
"golang.org/x/sync/singleflight"
"net"
"strings"
"time"
)
type GoIp struct {
dbReader *geoip2.Reader
dbLocation *ip2location.DB
IpServiceUrl string
singlefightGForIp singleflight.Group
}
func New(config Config) (*GoIp, error) {
g := &GoIp{
IpServiceUrl: config.IpServiceUrl,
singlefightGForIp: singleflight.Group{},
}
if config.Mmdb != "" {
db, err := geoip2.Open(config.Mmdb)
if err != nil {
return nil, err
}
g.dbReader = db
}
if config.Ip2locationDB != "" {
dbLocation, err := ip2location.OpenDB(config.Ip2locationDB)
if err != nil {
return nil, err
}
g.dbLocation = dbLocation
}
return g, nil
}
// 查询 IP相关信息,返回IP所属国家中文名称
func (g *GoIp) QueryDbReaderCountryForZhName(ipStr string) (*IpCountry, error) {
if g.dbReader == nil {
return nil, fmt.Errorf("dbReader is nil")
}
ip := net.ParseIP(ipStr)
recordCountry, err := g.dbReader.Country(ip)
if err != nil {
return nil, err
}
country := &IpCountry{
IsoCode: recordCountry.Country.IsoCode,
Name: recordCountry.Country.Names["zh-CN"],
Continent: recordCountry.Continent.Names["zh-CN"],
ContinentCode: recordCountry.Continent.Code,
}
return country, nil
}
// IP 相关信息
func (g *GoIp) QueryLocationInfoByIp(ipStr string) (*IP2Locationrecord, error) {
if g.dbLocation == nil {
return nil, fmt.Errorf("dbLocation is nil")
}
results, err := g.dbLocation.Get_all(ipStr)
if err != nil {
return nil, err
}
rsp := &IP2Locationrecord{
Country_short: results.Country_short,
Country_long: results.Country_long,
Region: results.Region,
City: results.City,
Isp: results.Isp,
Latitude: results.Latitude,
Longitude: results.Longitude,
Domain: results.Domain,
Zipcode: results.Zipcode,
Timezone: results.Timezone,
Netspeed: results.Netspeed,
Iddcode: results.Iddcode,
Areacode: results.Areacode,
Weatherstationcode: results.Weatherstationcode,
Weatherstationname: results.Weatherstationname,
Mcc: results.Mcc,
Mnc: results.Mnc,
Mobilebrand: results.Mobilebrand,
Elevation: results.Elevation,
Usagetype: results.Usagetype,
Addresstype: results.Addresstype,
Category: results.Category,
District: results.District,
Asn: results.Asn,
As: results.As,
}
return rsp, nil
}
func (g *GoIp) GetLocationInfoByIp(ip string, duration time.Duration) (*IpLocation, error) {
cachedIpInfo, ok := gocache.Default().Get(ip)
if ok {
return cachedIpInfo.(*IpLocation), nil
}
rst, err, _ := g.singlefightGForIp.Do(ip, func() (interface{}, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
ipInfo, err := Default().GetIpLocation(ctx, ip)
if err != nil {
return nil, err
}
ipInfo.IsoCode = strings.ToLower(ipInfo.IsoCode)
if ipInfo.IsoCode == "cn" {
//ipInfo.Isp = ChinaISPNameToPinyin(ipInfo.Isp)
}
gocache.Default().SharedCache.Set(ip, ipInfo, duration)
return ipInfo, nil
})
if err == nil {
return rst.(*IpLocation), nil
}
return nil, err
}
// ----------------------------------------------------------------
type IpLocation struct {
IsoCode string `json:"iso_code"` //iso 编码 https://zh.m.wikipedia.org/zh/ISO_3166-1
Name string `json:"name"`
Continent string `json:"continent"` //洲
ContinentCode string `json:"continent_code"` //洲编码
Isp string `json:"isp"` //isp
}
type ipLocationResp struct {
Code int64 `json:"code"`
Msg string `json:"msg"`
Data IpLocation `json:"data"`
}
func (g *GoIp) IsLocalIP(ip net.IP) bool {
if ip.IsLoopback() {
return true
}
return ip.IsPrivate()
}
// 获取IP地址的地理位置信息 需要提供IpServiceUrl
func (g *GoIp) GetIpLocation(ctx context.Context, ip string) (*IpLocation, error) {
netIP := net.ParseIP(ip)
if g.IsLocalIP(netIP) {
return &IpLocation{}, nil
}
request := &gohttp.Request{
Url: g.IpServiceUrl,
Timeout: time.Second * 5,
}
request.SetQueryParams("ip", ip)
gh := gohttp.GoHttp[ipLocationResp]{
Request: request,
Headers: map[string]string{"Content-Type": "application/json"},
}
rst, err := gh.HttpGet(ctx)
if err != nil {
return nil, fmt.Errorf("request error: %w", err)
}
if rst.Code == 0 {
return &rst.Data, nil
}
return nil, fmt.Errorf("request error: %w", err)
}
//---
go-ip/test/main.go (1.2 KiB)
package main
import (
"context"
goip "github.com/gif-gif/go.io/go-ip"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"time"
)
func main() {
goio.Init(goio.DEVELOPMENT)
config := goip.Config{
//Mmdb: "/Users/Jerry/Documents/my/dockers/projects/golang/ip_service/ip/data/GeoLite2-Country.mmdb",
Ip2locationDB: "/Users/Jerry/Documents/my/work/nodelink/doc/ip2location-IP-COUNTRY-REGION-CITY-ISP.BIN",
IpServiceUrl: "",
}
ip := "154.18.180.43"
err := goip.Init(config)
if err != nil {
golog.WithTag("goip").Error(err.Error())
return
}
//ipZhInfo, err := goip.Default().QueryDbReaderCountryForZhName(ip)
//if err != nil {
// golog.WithTag("goip").Error(err.Error())
// return
//}
//
//golog.WithTag("goip").Info(ipZhInfo)
ipInfo, err := goip.Default().QueryLocationInfoByIp(ip)
if err != nil {
golog.WithTag("goip").Error(err.Error())
return
}
golog.WithTag("goip").Info(ipInfo)
if config.IpServiceUrl != "" {
ipinfo, err := goip.Default().GetIpLocation(context.Background(), ip)
if err != nil {
golog.WithTag("goip").Error(err.Error())
return
}
golog.WithTag("goip").Info(ipinfo)
}
time.Sleep(time.Second * 5)
}
go-ip/types.go (1.0 KiB)
package goip
type IpCountry struct {
IsoCode string `json:"iso_code"` //iso 编码 https://zh.m.wikipedia.org/zh/ISO_3166-1
Name string `json:"name"`
Continent string `json:"continent"` //洲
ContinentCode string `json:"continent_code"` //洲编码
Isp string `json:"isp"` //isp
}
type IP2Locationrecord struct {
Country_short string
Country_long string
Region string
City string
Isp string
Latitude float32
Longitude float32
Domain string
Zipcode string
Timezone string
Netspeed string
Iddcode string
Areacode string
Weatherstationcode string
Weatherstationname string
Mcc string
Mnc string
Mobilebrand string
Elevation float32
Usagetype string
Addresstype string
Category string
District string
Asn string
As string
}
go-job/gojob.go (7.7 KiB)
package gojob
import (
"fmt"
"github.com/go-co-op/gocron/v2"
"github.com/google/uuid"
"time"
)
type (
GoJob struct {
cron gocron.Scheduler
}
)
func New(options ...gocron.SchedulerOption) (*GoJob, error) {
c, err := gocron.NewScheduler(options...)
if err != nil {
return nil, err
}
o := &GoJob{
cron: c,
}
return o, nil
}
func (o *GoJob) Cron() gocron.Scheduler {
return o.cron
}
func (c *GoJob) Start() {
c.cron.Start()
}
func (c *GoJob) Stop() error {
return c.cron.Shutdown()
}
func (c *GoJob) RemoveJob(jobID uuid.UUID) error {
return c.cron.RemoveJob(jobID)
}
func (c *GoJob) NewJob(jobDefinition gocron.JobDefinition, task gocron.Task, options ...gocron.JobOption) (gocron.Job, error) {
return c.cron.NewJob(jobDefinition, task, options...)
}
//job options start
// 在某一时刻运行
// s, _ := NewScheduler()
// defer func() { _ = s.Shutdown() }()
//
// start := time.Date(9999, 9, 9, 9, 9, 9, 9, time.UTC)
//
// j, _ := s.NewJob(
//
// DurationJob(
// time.Second,
// ),
// NewTask(
// func(one string, two int) {
// fmt.Printf("%s, %d", one, two)
// },
// "one", 2,
// ),
// WithStartAt(
// WithStartDateTime(start),
// ),
//
// )
// s.Start()
//
// next, _ := j.NextRun()
// fmt.Println(next)
//
// _ = s.StopJobs()
//
// 定时执行启动 开始时间
func (c *GoJob) WithStartAt(start time.Time) gocron.JobOption {
//start := time.Date(9999, 9, 9, 9, 9, 9, 9, time.UTC)
return gocron.WithStartAt(
gocron.WithStartDateTime(start),
)
}
// interval 月频, 0-6-->周日 周一 ... 周六, hours 具体执行时间列表
func (c *GoJob) MonthlyJob(options *[]gocron.JobOption, interval uint, daysOfTheMonth []int, hours []uint, minute uint, fn any, parameters ...any) (gocron.Job, error) {
if options == nil {
options = &[]gocron.JobOption{}
}
hoursAtTime := []gocron.AtTime{}
for _, hour := range hours {
hoursAtTime = append(hoursAtTime, gocron.NewAtTime(hour, minute, 0))
}
return c.NewJob(
gocron.MonthlyJob(
interval,
gocron.NewDaysOfTheMonth(daysOfTheMonth[0], daysOfTheMonth[1:]...),
gocron.NewAtTimes(
gocron.NewAtTime(hours[0], minute, 0),
hoursAtTime...,
),
),
gocron.NewTask(
fn,
parameters,
),
*options...,
)
}
// interval 周频, 0-6-->周日 周一 ... 周六, hours 具体执行时间列表
func (c *GoJob) WeeklyJob(options *[]gocron.JobOption, interval uint, daysOfTheWeek []time.Weekday, hours []uint, minutes uint, fn any, parameters ...any) (gocron.Job, error) {
if options == nil {
options = &[]gocron.JobOption{}
}
hoursAtTime := []gocron.AtTime{}
for _, hour := range hours {
hoursAtTime = append(hoursAtTime, gocron.NewAtTime(hour, minutes, 0))
}
return c.NewJob(
gocron.WeeklyJob(
interval,
gocron.NewWeekdays(daysOfTheWeek[0], daysOfTheWeek[1:]...),
gocron.NewAtTimes(
gocron.NewAtTime(hours[0], minutes, 0),
hoursAtTime...,
),
),
gocron.NewTask(
fn,
parameters...,
),
*options...,
)
}
// 当前时间 seconds 秒之后执行一次
func (c *GoJob) OneTimeJobForSeconds(options *[]gocron.JobOption, seconds uint, fn any, parameters ...any) (gocron.Job, error) {
if options == nil {
options = &[]gocron.JobOption{}
}
return c.NewJob(
gocron.OneTimeJob(
gocron.OneTimeJobStartDateTime(time.Now().Add(time.Duration(seconds)*time.Second)),
),
gocron.NewTask(
fn,
parameters...,
),
*options...,
)
}
// 当前时间 minute 分钟之后执行一次
func (c *GoJob) OneTimeJobForMinute(options *[]gocron.JobOption, minute uint, fn any, parameters ...any) (gocron.Job, error) {
if options == nil {
options = &[]gocron.JobOption{}
}
return c.NewJob(
gocron.OneTimeJob(
gocron.OneTimeJobStartDateTime(time.Now().Add(time.Duration(minute)*time.Minute)),
),
gocron.NewTask(
fn,
parameters...,
),
*options...,
)
}
// 每天定时执行
func (c *GoJob) DailyJob(options *[]gocron.JobOption, interval uint, hours []uint, minute uint, fn any, parameters ...any) (gocron.Job, error) {
if options == nil {
options = &[]gocron.JobOption{}
}
hoursAtTime := []gocron.AtTime{}
for _, hour := range hours {
hoursAtTime = append(hoursAtTime, gocron.NewAtTime(hour, minute, 0))
}
return c.NewJob(
gocron.DailyJob(interval, gocron.NewAtTimes(
gocron.NewAtTime(hours[0], minute, 0),
hoursAtTime...,
)),
gocron.NewTask(
fn,
parameters...,
),
*options...,
)
}
// 隔多少秒执行
func (c *GoJob) DurationJob(options *[]gocron.JobOption, seconds int, fn any, parameters ...any) (gocron.Job, error) {
if options == nil {
options = &[]gocron.JobOption{}
}
return c.NewJob(
gocron.DurationJob(
time.Duration(seconds)*time.Second,
),
gocron.NewTask(
fn,
parameters...,
),
//gocron.WithSingletonMode(gocron.LimitModeReschedule),
*options...,
)
}
// DurationRandomJob 定义一个新作业,该作业以提供的最小和最大持续时间值之间的随机间隔运行
func (c *GoJob) DurationRandomJob(options *[]gocron.JobOption, minDuration, maxDuration time.Duration, function any, parameters ...any) (gocron.Job, error) {
if options == nil {
options = &[]gocron.JobOption{}
}
return c.NewJob(
gocron.DurationRandomJob(
minDuration, maxDuration,
),
gocron.NewTask(
function,
parameters...,
),
*options...,
)
}
// spec is crontab pattern crontab 表达式
func (c *GoJob) CronJob(spec string, options *[]gocron.JobOption, function any, parameters ...any) (gocron.Job, error) {
if options == nil {
options = &[]gocron.JobOption{}
}
return c.NewJob(
gocron.CronJob(
// standard cron tab parsing
spec,
true, //六位crontab 规则时true,带秒位
),
gocron.NewTask(
function,
parameters...,
),
*options...,
)
}
// crontab 每天0点0分0秒执行
func (c *GoJob) Day(options *[]gocron.JobOption, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob("0 0 0 * * *", options, fn, parameters...)
}
// crontab 每天x点0分0秒执行
func (c *GoJob) DayHour(options *[]gocron.JobOption, hour int, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob(fmt.Sprintf("0 0 %d * * *", hour), options, fn, parameters...)
}
// crontab 每天每x点每x分0秒执行
func (c *GoJob) DayHourMinute(options *[]gocron.JobOption, hour, minute int, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob(fmt.Sprintf("0 %d %d * * *", minute, hour), options, fn, parameters...)
}
// crontab 每小时执行
func (c *GoJob) Hour(options *[]gocron.JobOption, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob("0 0 */1 * * *", options, fn, parameters...)
}
// crontab 每小时minute分执行
func (c *GoJob) HourMinute(options *[]gocron.JobOption, minute int, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob(fmt.Sprintf("0 %d */1 * * *", minute), options, fn, parameters...)
}
// crontab 每隔x小时执行
func (c *GoJob) HourX(options *[]gocron.JobOption, x int, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob(fmt.Sprintf("0 0 */%d * * *", x), options, fn, parameters...)
}
// crontab 每分钟执行
func (c *GoJob) Minute(options *[]gocron.JobOption, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob("0 */1 * * * *", options, fn, parameters...)
}
// crontab 每隔x分钟执行
func (c *GoJob) MinuteX(options *[]gocron.JobOption, x int, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob(fmt.Sprintf("0 */%d * * * *", x), options, fn, parameters...)
}
// crontab 每秒钟执行
func (c *GoJob) Second(options *[]gocron.JobOption, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob("* * * * * *", options, fn, parameters...)
}
// crontab 每隔x秒执行
func (c *GoJob) SecondX(options *[]gocron.JobOption, x int, fn any, parameters ...any) (gocron.Job, error) {
return c.CronJob(fmt.Sprintf("*/%d * * * * *", x), options, fn, parameters...)
}
go-job/readme.md (2.4 KiB)
定时任务
- 基于库: https://github.com/go-co-op/gocron/
- 官方文档: https://pkg.go.dev/github.com/go-co-op/gocron/v2
package main
import (
"fmt"
gojob "github.com/gif-gif/go.io/go-job"
golog "github.com/gif-gif/go.io/go-log"
"github.com/go-co-op/gocron/v2"
"github.com/gogf/gf/util/gconv"
"github.com/google/uuid"
"time"
)
func main() {
n := 0
cron, err := gojob.New()
if err != nil {
golog.WithTag("gojob").Error(err)
}
defer cron.Stop()
cron.Start()
job, err := cron.DurationJob(&[]gocron.JobOption{
gocron.WithLimitedRuns(2), //最大执行次数
gocron.WithSingletonMode(gocron.LimitModeWait), // 限制重叠执行
gocron.WithStartAt(gocron.WithStartImmediately()), //马上开始
gocron.WithEventListeners(
gocron.AfterJobRunsWithError(
func(jobID uuid.UUID, jobName string, err error) {
golog.WithTag("AfterJobRunsWithError-gojob").Error(jobID, jobName, err.Error())
},
),
gocron.AfterJobRunsWithPanic(
func(jobID uuid.UUID, jobName string, err any) {
golog.WithTag("AfterJobRunsWithPanic-gojob").Error(jobID, jobName, err)
},
),
gocron.AfterLockError(func(jobID uuid.UUID, jobName string, err error) {
golog.WithTag("AfterLockError-gojob").Error(jobID, jobName, err.Error())
}),
),
}, 1, func(nn int) error {
golog.WithTag("gojobStart").Info("testing->" + gconv.String(nn))
time.Sleep(time.Second * 5)
golog.WithTag("gojobEnd").Info("testing->" + gconv.String(nn))
a := 1 / nn //test for panic
return fmt.Errorf("gojobEnd failed" + gconv.String(a)) //test for error
}, n)
if err != nil {
golog.WithTag("gojob").Error(err)
} else {
golog.WithTag("gojob").Info("job.ID:" + job.ID().String())
}
time.Sleep(time.Second * 500)
golog.InfoF("end of gojob")
}
func simpleUseGoJob() {
n := 0
cron, err := gojob.New()
if err != nil {
golog.WithTag("gojob").Error(err)
}
defer cron.Stop()
cron.Start()
job, err := cron.DurationJob(nil, 1, func(nn int) error {
golog.WithTag("gojobStart").Info("testing->" + gconv.String(nn))
time.Sleep(time.Second * 3)
golog.WithTag("gojobEnd").Info("testing->" + gconv.String(nn))
return nil
}, n)
if err != nil {
golog.WithTag("gojob").Error(err)
} else {
golog.WithTag("gojob").Info("job.ID:" + job.ID().String())
}
time.Sleep(time.Second * 500)
golog.InfoF("end of gojob")
}
go-job/test/main.go (2.5 KiB)
package main
import (
"fmt"
gocontext "github.com/gif-gif/go.io/go-context"
gojob "github.com/gif-gif/go.io/go-job"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/goio"
"github.com/go-co-op/gocron/v2"
"github.com/gogf/gf/util/gconv"
"github.com/google/uuid"
"time"
)
func main() {
goio.Init(goio.DEVELOPMENT)
//testJob()
simpleUseGoJob()
}
func testJob() {
n := 0
cron, err := gojob.New()
if err != nil {
golog.WithTag("gojob").Error(err)
}
defer cron.Stop()
cron.Start()
job, err := cron.DurationJob(&[]gocron.JobOption{
gocron.WithLimitedRuns(20), //最大执行次数
gocron.WithSingletonMode(gocron.LimitModeWait), // 限制重叠执行
gocron.WithStartAt(gocron.WithStartImmediately()), //马上开始
gocron.WithEventListeners(
gocron.AfterJobRunsWithError(
func(jobID uuid.UUID, jobName string, err error) {
golog.WithTag("AfterJobRunsWithError-gojob").Error(jobID, jobName, err.Error())
cron.Stop()
},
),
gocron.AfterJobRunsWithPanic(
func(jobID uuid.UUID, jobName string, err any) {
golog.WithTag("AfterJobRunsWithPanic-gojob").Error(jobID, jobName, err)
cron.Stop()
},
),
gocron.AfterLockError(func(jobID uuid.UUID, jobName string, err error) {
golog.WithTag("AfterLockError-gojob").Error(jobID, jobName, err.Error())
cron.Stop()
}),
),
}, 1, func(nn int) error {
golog.WithTag("gojobStart").Info("testing->" + gconv.String(nn))
time.Sleep(time.Second * 5)
golog.WithTag("gojobEnd").Info("testing->" + gconv.String(nn))
a := 1 / nn //test for panic
return fmt.Errorf("gojobEnd failed" + gconv.String(a)) //test for error
}, n)
if err != nil {
golog.WithTag("gojob").Error(err)
} else {
golog.WithTag("gojob").Info("job.ID:" + job.ID().String())
}
time.Sleep(time.Second * 500)
golog.InfoF("end of gojob")
}
func simpleUseGoJob() {
cron, err := gojob.New()
if err != nil {
golog.WithTag("gojob").Error(err)
}
defer cron.Stop()
cron.Start()
cron.HourMinute(nil, 10, func() error {
golog.WithTag("gojob").Info("testing->----------")
return nil
})
//
//job, err := cron.SecondX(nil, 1, func(nn int) error {
// golog.WithTag("gojob").Info("testing->" + gconv.String(nn))
// return nil
//}, n)
//
//if err != nil {
// golog.WithTag("gojob").Error(err)
//} else {
// golog.WithTag("gojob").Info("job.ID:" + job.ID().String())
//}
<-gocontext.WithCancel().Done()
golog.InfoF("end of gojob")
}
go-lock/golock.go (1.5 KiB)
package golock
import "sync"
// RWMutex 也称为读写互斥锁,读写互斥锁就是读取/写入互相排斥的锁。它可以由任意数量的读取操作的 goroutine 或单个写入操作的 goroutine 持有。
// 读写互斥锁 RWMutex 类型有五个方法,Lock,Unlock,Rlock,RUnlock 和 RLocker。其中,RLocker 返回一个 Locker 接口,
// 该接口通过调用 rw.RLock 和 rw.RUnlock 来实现 Lock 和 Unlock 方法。
// 不能拷贝锁
type GoLock struct {
Mutex sync.Mutex // 读锁时不能写,写锁时不能读取
MuteRW sync.RWMutex //读写互斥锁,并发读取,单一写入。读多写少性能会好
}
func New() *GoLock {
return &GoLock{}
}
// 共享内存加锁,同步锁
func (g *GoLock) LockFunc(fn func(parameters ...any), parameters ...any) {
g.Lock()
defer g.Unlock()
fn(parameters...)
}
// 加读锁
func (g *GoLock) RLockFunc(fn func(parameters ...any), parameters ...any) {
g.RLock()
defer g.RUnlock()
fn(parameters...)
}
// 加写锁
func (g *GoLock) WLockFunc(fn func(parameters ...any), parameters ...any) {
g.WLock()
defer g.WUnlock()
fn(parameters...)
}
// 获取同步锁
func (g *GoLock) Lock() {
g.Mutex.Lock()
}
// 释放同步锁
func (g *GoLock) Unlock() {
g.Mutex.Unlock()
}
// 获取读锁
func (g *GoLock) RLock() {
g.MuteRW.RLock()
}
// 释放读锁
func (g *GoLock) RUnlock() {
g.MuteRW.RUnlock()
}
// 获取写锁
func (g *GoLock) WLock() {
g.MuteRW.Lock()
}
// 释放取写锁
func (g *GoLock) WUnlock() {
g.MuteRW.Unlock()
}
go-lock/readme.md (970 B)
Go-LOCK
- 用法
package main
import (
"fmt"
golock "github.com/gif-gif/go.io/go-lock"
gopool "github.com/gif-gif/go.io/go-pool"
"github.com/gif-gif/go.io/goio"
)
func main() {
goio.Init(goio.DEVELOPMENT)
testGoPool()
//testSyncLock()
}
func testGoPool() {
var (
count int
)
lock := golock.New()
//并发池
gp := gopool.NewFixedSizePool(10, 10)
defer gp.StopAndWait()
group := gp.NewTaskGroup()
for i := 0; i < 10; i++ {
group.Submit(func() {
for i := 1000; i > 0; i-- {
lock.WLockFunc(func(parameters ...any) {
count++
})
}
fmt.Println(count)
})
}
group.Wait()
}
func testSyncLock() {
var (
count int
)
lock := golock.New()
for i := 0; i < 2; i++ {
go func() {
for i := 1000; i > 0; i-- {
//lock.WLock()
//count++
//lock.WUnlock()
lock.WLockFunc(func(parameters ...any) {
count++
})
}
fmt.Println(count)
}()
}
fmt.Scanf("\n") //等待子线程全部结束
}
go-lock/test/main.go (619 B)
package main
import (
"fmt"
golock "github.com/gif-gif/go.io/go-lock"
gopool "github.com/gif-gif/go.io/go-pool"
"github.com/gif-gif/go.io/goio"
)
func main() {
goio.Init(goio.DEVELOPMENT)
testSyncLock()
}
func testSyncLock() {
var (
count int
)
lock := golock.New()
//并发池
gp := gopool.NewFixedSizePool(10, 10) // 可以并发10个
defer gp.StopAndWait()
group := gp.NewTaskGroup()
for i := 0; i < 2; i++ {
group.Submit(func() {
for i := 100000; i > 0; i-- {
lock.WLockFunc(func(parameters ...any) {
count++
})
}
})
}
group.Wait()
fmt.Println(count) //输出 200000
}
go-log/adapter.go (63 B)
package golog
type Adapter interface {
Write(msg *Message)
}
go-log/adapters/es.go (862 B)
package adapters
import (
goes "github.com/gif-gif/go.io/go-db/go-es"
"github.com/gif-gif/go.io/go-log"
"sync"
)
type EsAdapter struct {
index string
id string
opt goes.Config
mu sync.Mutex
}
func NewEsLog(index, id string, opt goes.Config) *golog.Logger {
return golog.New(NewEsAdapter(index, id, opt))
}
func NewEsAdapter(index, id string, opt goes.Config) *EsAdapter {
err := goes.Init(opt)
if err != nil {
return nil
}
fa := &EsAdapter{
index: index,
id: id,
opt: opt,
}
return fa
}
func (fa *EsAdapter) Write(msg *golog.Message) {
client := goes.GetClient(fa.opt.Name)
if client == nil {
return
}
_, err := goes.GetClient(fa.opt.Name).DocCreate(fa.index, fa.id, msg.MAP())
if err != nil {
return
}
}
func (fa *EsAdapter) closeEs() {
client := goes.GetClient(fa.opt.Name)
if client == nil {
return
}
}
go-log/adapters/file.go (4.5 KiB)
package adapters
import (
"fmt"
gocontext "github.com/gif-gif/go.io/go-context"
"github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gif-gif/go.io/go-utils/gotime"
"log"
"os"
"path/filepath"
"regexp"
"runtime"
"sync"
"time"
)
type FileAdapter struct {
filepath string
filename string
fh *os.File
maxSize int64
count int
keepDays int
ch chan []byte
mu sync.Mutex
timer *gotime.Timer
}
func NewFileLog(opts ...FileOption) *golog.Logger {
return golog.New(NewFileAdapter(opts...))
}
func NewFileAdapter(opt ...FileOption) *FileAdapter {
opts := defaultFileOptions
for _, o := range opt {
o.apply(&opts)
}
fa := &FileAdapter{
filepath: opts.Filepath,
maxSize: opts.MaxSize,
keepDays: opts.KeepDays,
ch: make(chan []byte, runtime.NumCPU()*2),
}
if l := len(fa.filepath); fa.filepath[l-1:] != "/" {
fa.filepath += "/"
}
if _, err := os.Stat(fa.filepath); os.IsNotExist(err) {
os.MkdirAll(fa.filepath, 0755)
}
var (
nw = time.Now()
ymd = nw.Format("20060102")
)
files, _ := filepath.Glob(fa.filepath + ymd + "_*.log")
fa.count = len(files)
fa.timer = gotime.NewTimer(opts.ClearLogInterval) // 每12点清理一次
fa.timer.Start(func() {
fa.CleanOldLogs()
})
goutils.AsyncFunc(func() {
select {
case <-gocontext.WithCancel().Done():
fa.timer.Stop()
fmt.Println("日志文件清理定时器已停止")
}
})
return fa
}
func (fa *FileAdapter) Write(msg *golog.Message) {
go func() {
defer func() {
if r := recover(); r != nil {
log.Println(r)
}
}()
c := <-fa.ch
fa.writeHandle(c)
}()
fa.ch <- msg.JSON()
}
func (fa *FileAdapter) writeHandle(b []byte) {
fa.mu.Lock()
defer func() { fa.mu.Unlock() }()
var (
nw = time.Now()
ymd = nw.Format("20060102")
filename = ymd + ".log"
)
if filename != fa.filename {
fa.filename = filename
fa.closeFile()
}
if fa.fh == nil {
if err := fa.openFile(); err != nil {
return
}
}
if fa.maxSize > 0 {
if err := fa.cutFile(ymd); err != nil {
return
}
}
if fa.fh != nil {
fa.fh.Write(b)
fa.fh.Write([]byte("\n"))
}
}
func (fa *FileAdapter) closeFile() {
if fa.fh == nil {
return
}
fa.fh.Close()
fa.fh = nil
}
func (fa *FileAdapter) openFile() (err error) {
fa.fh, err = os.OpenFile(fa.filepath+fa.filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
log.Println(err.Error())
}
return
}
func (fa *FileAdapter) cutFile(ymd string) (err error) {
var info os.FileInfo
info, err = os.Stat(fa.filepath + fa.filename)
if err != nil {
log.Println(err.Error())
return
}
if info.Size() < fa.maxSize {
return
}
fa.count++
filename := fmt.Sprintf("%s_%d.log", fa.filepath+ymd, fa.count)
if err = os.Rename(fa.filepath+fa.filename, filename); err != nil {
log.Println(err.Error())
return
}
if fa.fh != nil {
fa.fh.Close()
fa.fh = nil
}
if err = fa.openFile(); err != nil {
return
}
return
}
func (fa *FileAdapter) CleanOldLogs() error {
// 获取当前时间
now := time.Now()
// 计算截止时间(7天前)
cutoffTime := now.AddDate(0, 0, -fa.keepDays)
// 读取目录中的所有文件
files, err := os.ReadDir(fa.filepath)
if err != nil {
return fmt.Errorf("读取目录失败: %v", err)
}
// 日志文件名正则表达式
// 匹配格式: 20250525.log 或 20250528_1.log
logPattern := regexp.MustCompile(`^(\d{8})(_\d+)?\.log$`)
deletedCount := 0
keptCount := 0
for _, file := range files {
// 跳过目录
if file.IsDir() {
continue
}
fileName := file.Name()
// 检查是否是日志文件
matches := logPattern.FindStringSubmatch(fileName)
if matches == nil {
continue
}
// 解析日期
dateStr := matches[1]
fileDate, err := time.Parse("20060102", dateStr)
if err != nil {
//log.Printf("解析日期失败 %s: %v", fileName, err)
continue
}
// 获取文件的完整路径
filePath := filepath.Join(fa.filepath, fileName)
// 判断是否需要删除(文件日期早于截止日期)
if fileDate.Before(cutoffTime) {
// 删除文件
if err := os.Remove(filePath); err != nil {
//log.Printf("删除文件失败 %s: %v", filePath, err)
} else {
deletedCount++
//log.Printf("已删除: %s (日期: %s)", fileName, fileDate.Format("2006-01-02"))
}
} else {
keptCount++
//log.Printf("保留: %s (日期: %s)", fileName, fileDate.Format("2006-01-02"))
}
}
golog.InfoF("清理完成: 删除 %d 个文件, 保留 %d 个文件", deletedCount, keptCount)
return nil
}
go-log/adapters/file_options.go (1.2 KiB)
package adapters
import "time"
var defaultFileOptions = fileOptions{
Filepath: "logs/",
MaxSize: 1 << 29,
KeepDays: 7, //日志保留7天
ClearLogInterval: time.Hour * 12, //每12小清理一次
}
type fileOptions struct {
Filepath string
MaxSize int64
KeepDays int
ClearLogInterval time.Duration
}
type FileOption interface {
apply(*fileOptions)
}
type funcFileOption struct {
f func(*fileOptions)
}
func (f *funcFileOption) apply(options *fileOptions) {
f.f(options)
}
func newFuncFileOption(f func(*fileOptions)) *funcFileOption {
return &funcFileOption{
f: f,
}
}
func FilePathOption(filepath string) FileOption {
return newFuncFileOption(func(options *fileOptions) {
options.Filepath = filepath
})
}
func FileMaxSizeOption(maxSize int64) FileOption {
return newFuncFileOption(func(options *fileOptions) {
options.MaxSize = maxSize
})
}
func KeepDaysOption(keepDays int) FileOption {
return newFuncFileOption(func(options *fileOptions) {
options.KeepDays = keepDays
})
}
func ClearLogIntervalOption(clearLogInterval time.Duration) FileOption {
return newFuncFileOption(func(options *fileOptions) {
options.ClearLogInterval = clearLogInterval
})
}
go-log/adapters/kafka.go (958 B)
package adapters
import (
"github.com/gif-gif/go.io/go-log"
gokafka "github.com/gif-gif/go.io/go-mq/go-kafka"
"sync"
)
type KafkaAdapter struct {
topic string
opt gokafka.Config
mu sync.Mutex
}
func NewKafkaLog(topic string, opt gokafka.Config) *golog.Logger {
return golog.New(NewKafkaAdapter(topic, opt))
}
func NewKafkaAdapter(topic string, opt gokafka.Config) *KafkaAdapter {
err := gokafka.Init(opt)
if err != nil {
return nil
}
fa := &KafkaAdapter{
topic: topic,
opt: opt,
}
return fa
}
func (fa *KafkaAdapter) Write(msg *golog.Message) {
client := gokafka.GetClient(fa.opt.Name)
if client == nil {
return
}
//err := client.Producer().SendAsyncMessage(msg.JSON(), func(msg *gokafka.ProducerMessage, err error) {
//
//})
//
//if err != nil {
// log.Println(err.Error())
//}
}
func (fa *KafkaAdapter) CloseKafka() {
client := gokafka.GetClient(fa.opt.Name)
if client == nil {
return
}
client.Close()
}
go-log/console.go (1.2 KiB)
package golog
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
)
type ConsoleAdapter struct {
}
func NewConsoleLog() *Logger {
return New(&ConsoleAdapter{})
}
func (ca *ConsoleAdapter) Write(msg *Message) {
var (
buf bytes.Buffer
nw = time.Now()
level = LevelText[msg.Level]
color = Colors[msg.Level]
)
buf.WriteString(nw.Format("2006-01-02 15:04:05"))
buf.WriteString(" ")
buf.WriteString(color(fmt.Sprintf("%-5s", level)))
buf.WriteString(" ")
if l := len(msg.Entry.Tags); l > 0 {
buf.WriteString("[" + strings.Join(msg.Entry.Tags, "][") + "]")
buf.WriteString(" ")
}
if l := len(msg.Message); l > 0 {
for _, v := range msg.Message {
buf.WriteString(fmt.Sprint(v))
buf.WriteString(" ")
}
}
if l := len(msg.Entry.Data); l > 0 {
data := map[string]interface{}{}
for _, i := range msg.Entry.Data {
data[i.Field] = i.Value
}
if b, err := json.Marshal(&data); err == nil {
buf.Write(b)
}
buf.WriteString(" ")
}
if l := len(msg.Trace); l > 0 {
if b, err := json.Marshal(&msg.Trace); err == nil {
buf.Write(b)
}
buf.WriteString(" ")
}
ca.writer().Write(append(buf.Bytes(), '\n'))
}
func (ca ConsoleAdapter) writer() io.Writer {
return os.Stdout
}
go-log/entry.go (3.3 KiB)
package golog
import (
"fmt"
"log"
"runtime"
"strings"
"time"
)
type Entry struct {
Tags []string
Data []DataField
msg *Message
l *Logger
}
type DataField struct {
Field string
Value interface{}
}
func NewEntry(l *Logger) *Entry {
return &Entry{l: l}
}
func (entry *Entry) WithTag(tags ...string) *Entry {
entry.Tags = append(entry.Tags, tags...)
return entry
}
func (entry *Entry) WithField(field string, value interface{}) *Entry {
entry.Data = append(entry.Data, DataField{Field: field, Value: value})
return entry
}
func (entry *Entry) Debug(v ...interface{}) {
entry.output(DEBUG, v...)
}
func (entry *Entry) DebugF(format string, v ...interface{}) {
entry.output(DEBUG, fmt.Sprintf(format, v...))
}
func (entry *Entry) Info(v ...interface{}) {
entry.output(INFO, v...)
}
func (entry *Entry) InfoF(format string, v ...interface{}) {
entry.output(INFO, fmt.Sprintf(format, v...))
}
func (entry *Entry) Warn(v ...interface{}) {
entry.output(WARN, v...)
}
func (entry *Entry) WarnF(format string, v ...interface{}) {
entry.output(WARN, fmt.Sprintf(format, v...))
}
func (entry *Entry) Error(v ...interface{}) {
entry.output(ERROR, v...)
}
func (entry *Entry) ErrorF(format string, v ...interface{}) {
entry.output(ERROR, fmt.Errorf(format, v...).Error())
}
func (entry *Entry) Panic(v ...interface{}) {
entry.output(PANIC, v...)
}
func (entry *Entry) PanicF(format string, v ...interface{}) {
entry.output(PANIC, fmt.Sprintf(format, v...))
}
func (entry *Entry) Fatal(v ...interface{}) {
entry.output(FATAL, v...)
}
func (entry *Entry) FatalF(format string, v ...interface{}) {
entry.output(FATAL, fmt.Sprintf(format, v...))
}
func (entry *Entry) output(level Level, v ...interface{}) {
entry.msg = &Message{
Level: level,
Message: v,
Time: time.Now(),
Entry: entry,
}
if level >= WARN {
entry.msg.Trace = entry.trace()
}
for _, fn := range entry.l.hooks {
go entry.hookHandler(fn)
}
if entry.l.adapter != nil {
entry.l.adapter.Write(entry.msg)
}
}
func (entry *Entry) hookHandler(fn func(msg *Message)) {
defer func() {
if r := recover(); r != nil {
log.Println(r)
}
}()
fn(entry.msg)
}
// runtime.Caller 仅能获取非 goroutine 的信息
func (entry *Entry) trace() (arr []string) {
arr = []string{}
for i := 3; i < 16; i++ {
_, file, line, _ := runtime.Caller(i)
if file == "" {
continue
}
if strings.Contains(file, ".pb.go") ||
strings.Contains(file, "runtime/") ||
strings.Contains(file, "src/") ||
strings.Contains(file, "pkg/mod/") ||
strings.Contains(file, "vendor/") ||
strings.Contains(file, "go-log") {
continue
}
arr = append(arr, fmt.Sprintf("%s %dL", entry.prettyFile(file), line))
}
return
}
func (entry *Entry) prettyFile(file string) string {
var (
index int
index2 int
)
if index = strings.LastIndex(file, "src/test/"); index >= 0 {
return file[index+9:]
}
if index = strings.LastIndex(file, "src/"); index >= 0 {
return file[index+4:]
}
if index = strings.LastIndex(file, "pkg/mod/"); index >= 0 {
return file[index+8:]
}
if index = strings.LastIndex(file, "vendor/"); index >= 0 {
return file[index+7:]
}
if index = strings.LastIndex(file, "/"); index < 0 {
return file
}
if index2 = strings.LastIndex(file[:index], "/"); index2 < 0 {
return file[index+1:]
}
return file[index2+1:]
}
go-log/level.go (676 B)
package golog
type Level int
type brush func(string) string
const (
DEBUG Level = iota
INFO
WARN
ERROR
PANIC
FATAL
)
var (
LevelText = map[Level]string{
DEBUG: "DEBUG",
INFO: "INFO",
WARN: "WARN",
ERROR: "ERROR",
PANIC: "PANIC",
FATAL: "FATAL",
}
Colors = map[Level]brush{
DEBUG: newBrush("1;34"), // blue
INFO: newBrush("1;32"), // green
WARN: newBrush("1;33"), // yellow
ERROR: newBrush("1;31"), // red
PANIC: newBrush("1;37"), // white
FATAL: newBrush("1;35"), // magenta
}
)
func newBrush(color string) brush {
pre := "\033["
reset := "\033[0m"
return func(text string) string {
return pre + color + "m" + text + reset
}
}
go-log/log.go (1.3 KiB)
package golog
import (
"sync"
)
var (
__log *Logger
__once sync.Once
)
func Default() *Logger {
__once.Do(func() {
__log = NewConsoleLog()
})
return __log
}
func SetAdapter(adapter Adapter) {
Default().adapter = adapter
}
func WithHook(fns ...func(msg *Message)) {
Default().WithHook(fns...)
}
func WithTag(tags ...string) *Entry {
return Default().WithTag(tags...)
}
func WithField(field string, value interface{}) *Entry {
return Default().WithField(field, value)
}
func Debug(v ...interface{}) {
Default().Debug(v...)
}
func DebugF(format string, v ...interface{}) {
Default().DebugF(format, v...)
}
func Info(v ...interface{}) {
Default().Info(v...)
}
func InfoF(format string, v ...interface{}) {
Default().InfoF(format, v...)
}
func Warn(v ...interface{}) {
Default().Warn(v...)
}
func WarnF(format string, v ...interface{}) {
Default().WarnF(format, v...)
}
func Error(v ...interface{}) {
Default().Error(v...)
}
func ErrorF(format string, v ...interface{}) {
Default().ErrorF(format, v...)
}
func Panic(v ...interface{}) {
Default().Panic(v...)
}
func PanicF(format string, v ...interface{}) {
Default().PanicF(format, v...)
}
func Fatal(v ...interface{}) {
Default().Fatal(v...)
}
func FatalF(format string, v ...interface{}) {
Default().FatalF(format, v...)
}
go-log/logger.go (1.5 KiB)
package golog
type Logger struct {
hooks []func(msg *Message)
adapter Adapter
}
func New(adapter Adapter) *Logger {
return &Logger{
adapter: adapter,
}
}
func (l *Logger) SetAdapter(adapter Adapter) {
l.adapter = adapter
}
func (l *Logger) WithHook(fns ...func(msg *Message)) {
l.hooks = append(l.hooks, fns...)
}
func (l *Logger) WithTag(tags ...string) *Entry {
return NewEntry(l).WithTag(tags...)
}
func (l *Logger) WithField(field string, value interface{}) *Entry {
return NewEntry(l).WithField(field, value)
}
func (l *Logger) Debug(v ...interface{}) {
NewEntry(l).Debug(v...)
}
func (l *Logger) DebugF(format string, v ...interface{}) {
NewEntry(l).DebugF(format, v...)
}
func (l *Logger) Info(v ...interface{}) {
NewEntry(l).Info(v...)
}
func (l *Logger) InfoF(format string, v ...interface{}) {
NewEntry(l).InfoF(format, v...)
}
func (l *Logger) Warn(v ...interface{}) {
NewEntry(l).Warn(v...)
}
func (l *Logger) WarnF(format string, v ...interface{}) {
NewEntry(l).WarnF(format, v...)
}
func (l *Logger) Error(v ...interface{}) {
NewEntry(l).Error(v...)
}
func (l *Logger) ErrorF(format string, v ...interface{}) {
NewEntry(l).ErrorF(format, v...)
}
func (l *Logger) Panic(v ...interface{}) {
NewEntry(l).Panic(v...)
}
func (l *Logger) PanicF(format string, v ...interface{}) {
NewEntry(l).PanicF(format, v...)
}
func (l *Logger) Fatal(v ...interface{}) {
NewEntry(l).Fatal(v...)
}
func (l *Logger) FatalF(format string, v ...interface{}) {
NewEntry(l).FatalF(format, v...)
}
go-log/message.go (970 B)
package golog
import (
"encoding/json"
"fmt"
"time"
)
type Message struct {
Level Level
Message []interface{}
Trace []string
Time time.Time
Entry *Entry
}
func (msg *Message) JSON() []byte {
buf, _ := json.Marshal(msg.MAP())
return buf
}
func (msg *Message) MAP() *map[string]interface{} {
data := map[string]interface{}{}
if l := len(msg.Entry.Data); l > 0 {
for _, i := range msg.Entry.Data {
msg.Message = append(msg.Message, fmt.Sprintf("%s=%s", i.Field, fmt.Sprint(i.Value)))
}
}
{
data["log_level"] = LevelText[msg.Level]
data["log_datetime"] = msg.Time.Format("2006-01-02 15:04:05")
if l := len(msg.Entry.Tags); l > 0 {
data["log_tags"] = msg.Entry.Tags
}
if l := len(msg.Message); l > 0 {
var arr []string
for _, i := range msg.Message {
arr = append(arr, fmt.Sprint(i))
}
data["log_message"] = arr
}
if l := len(msg.Trace); l > 0 {
data["log_trace"] = msg.Trace
}
}
return &data
}
go-log/readme.md (1.8 KiB)
日志级别
- Fatal 致命的
- Panic 严重的
- Error 错误的
- Warn 告警的
- Info 信息的
- Debug 调试的
文件说明
log.go对外开放的方法,默认console适配器,属于第1层级logger.golog对象,像SetAdapterWithHook是项目及全局方法,属于第2层级entry.go实体类,每次产生一条log时,都要new实体类,主要处理消息内容、标签、附加数据等,属于第3层级message.go消息内容对象,包装每条消息包含的字段信息adapter.go适配器console.go控制台适配器file.go文件适配器file_options.go文件适配器 选项
文件适配器
filepath日志保存目录filename日志文件名,使用yyyymmdd.logmaxSize文件大小最大值,默认512M,超过后,会切割文件FilePathOption()定义文件路径FileMaxSizeOption()定义文件大小最大值
输出对象(Adapters)
- console
- file
- kafka
- es
examples
- ConsoleTestExample:
l := NewConsoleLog()
l.WithHook(func(msg *Message) {
log.Println("this is hook", msg)
})
l.Debug("hi goio", "aa", 100)
l.DebugF("hi %s", "goio")
l.WithTag("u1").Debug("hi goio")
l.WithTag("u1", "u-1").Warn("hi goio")
l.WithTag("u1").WithField("name", "goio").Error("hi goio")
l.WithTag("u1").WithField("id", 101).Panic("hi goio")
l.WithTag("u1").WithField("id", 101).Fatal("hi goio")
- LogTest
//Default().SetAdapter(&FileAdapter{})
WithHook(func(msg *Message) {
log.Println("this is hook", msg)
})
Debug("hi goio", "aa", 100)
DebugF("hi %s", "goio")
WithTag("u1").Debug("hi goio")
WithTag("u1", "u-1").Debug("hi goio")
WithTag("u1").WithField("name", "goio").Debug("hi goio")
WithTag("u1").WithField("id", 101).Debug("hi goio")
WithTag("u1").WithField("id", 101).Debug("hi goio")
go-log/test/console_test.go (530 B)
package test
import (
"github.com/gif-gif/go.io/go-log"
"log"
"testing"
)
func TestNewConsoleLog(t *testing.T) {
l := golog.NewConsoleLog()
l.WithHook(func(msg *golog.Message) {
log.Println("this is hook", msg)
})
l.Debug("hi goio", "aa", 100)
l.DebugF("hi %s", "goio")
l.WithTag("u1").Debug("hi goio")
l.WithTag("u1", "u-1").Warn("hi goio")
l.WithTag("u1").WithField("name", "goio").Error("hi goio")
l.WithTag("u1").WithField("id", 101).Panic("hi goio")
l.WithTag("u1").WithField("id", 101).Fatal("hi goio")
}
go-log/test/file_test.go (830 B)
package test
import (
"github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/go-log/adapters"
"log"
"sync"
"testing"
"time"
)
func TestNewFileLog(t *testing.T) {
l := adapters.NewFileLog(
adapters.FilePathOption("logs/"),
adapters.FileMaxSizeOption(1<<20),
)
l.WithHook(func(msg *golog.Message) {
log.Println("this is hook", msg)
})
var wg sync.WaitGroup
for i := 0; i < 10000; i++ {
wg.Add(1)
go func() {
defer wg.Done()
l.Debug("hi goio", "aa", 100)
l.DebugF("hi %s", "goio")
l.WithTag("u1").Debug("hi goio")
l.WithTag("u1", "u-1").Warn("hi goio")
l.WithTag("u1").WithField("name", "goio").Error("hi goio")
l.WithTag("u1").WithField("id", 101).Panic("hi goio")
//l.WithTag("u1").WithField("id", 101).Fatal("hi goio")
}()
}
wg.Wait()
time.Sleep(3 * time.Second)
}
go-log/test/log_test.go (569 B)
package test
import (
"github.com/gif-gif/go.io/go-log"
"log"
"testing"
)
func TestDefault(t *testing.T) {
//Default().SetAdapter(&FileAdapter{})
golog.WithHook(func(msg *golog.Message) {
log.Println("this is hook", msg)
})
golog.Debug("hi goio", "aa", 100)
golog.DebugF("hi %s", "goio")
golog.WithTag("u1").Debug("hi goio")
golog.WithTag("u1", "u-1").Debug("hi goio")
golog.WithTag("u1").WithField("name", "goio").Debug("hi goio")
golog.WithTag("u1").WithField("id", 101).Debug("hi goio")
golog.WithTag("u1").WithField("id", 101).Debug("hi goio")
}
go-mail/config.go (318 B)
package gomail
type Config struct {
Username string `json:"username,optional" yaml:"Username"`
Password string `json:"password,optional" yaml:"Password"`
Host string `json:"host,optional" yaml:"Host"`
Port int `json:"port,optional" yaml:"Port"`
TLS bool `json:"tls,optional" yaml:"TLS"`
}
go-mail/default.go (145 B)
package gomail
var (
__mail iMail
)
func Init(conf Config) {
__mail = New(conf)
}
func Send(msg Message) error {
return __mail.Send(msg)
}
go-mail/mail.go (1.6 KiB)
package gomail
import (
"crypto/tls"
"fmt"
golog "github.com/gif-gif/go.io/go-log"
"io"
"net"
"net/smtp"
)
type iMail interface {
Send(msg Message) error
}
type mail struct {
conf Config
}
func New(conf Config) iMail {
return &mail{
conf: conf,
}
}
func (m *mail) Send(msg Message) (err error) {
if msg.Sender == "" {
msg.Sender = m.conf.Username
}
var (
conn net.Conn
cli *smtp.Client
)
conn, cli, err = m.client()
if err != nil {
return err
}
defer conn.Close()
defer cli.Close()
if err = cli.Auth(m.auth()); err != nil {
golog.Error(err.Error())
return
}
if err = cli.Mail(msg.Sender); err != nil {
golog.Error(err.Error())
return
}
for _, receiver := range msg.Receivers {
if err = cli.Rcpt(receiver); err != nil {
golog.Error(err.Error())
return
}
}
var (
w io.WriteCloser
)
if w, err = cli.Data(); err != nil {
golog.Error(err.Error())
return
}
defer w.Close()
if _, err = w.Write(msg.Html()); err != nil {
golog.Error(err.Error())
return
}
cli.Quit()
return
}
func (m *mail) client() (conn net.Conn, cli *smtp.Client, err error) {
addr := fmt.Sprintf("%s:%d", m.conf.Host, m.conf.Port)
if m.conf.TLS {
conn, err = tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true})
} else {
conn, err = net.Dial("tcp", addr)
}
if err != nil {
golog.Error(err.Error())
return
}
cli, err = smtp.NewClient(conn, m.conf.Host)
if err != nil {
golog.Error(err.Error())
return
}
return
}
func (m *mail) auth() (auth smtp.Auth) {
return smtp.PlainAuth("", m.conf.Username, m.conf.Password, m.conf.Host)
}
go-mail/mail_test.go (1.3 KiB)
package gomail
import "testing"
func TestMail_Send_126(t *testing.T) {
conf := Config{
Username: "[email protected]",
Password: "XYQNHPYFSLHYVRGCA",
Host: "smtp.126.com",
Port: 465,
TLS: true,
}
msg := Message{
Sender: "[email protected]",
SenderName: "测试者",
Receivers: []string{"[email protected]"},
Subject: "hi",
Body: "hi goio",
}
New(conf).Send(msg)
}
func TestMail_Send_qq(t *testing.T) {
conf := Config{
Username: "[email protected]",
Password: "k27Cicaftj9Nqp5da",
Host: "smtp.exmail.qq.com",
Port: 465,
TLS: true,
}
msg := Message{
Sender: "[email protected]",
SenderName: "测试者",
Receivers: []string{"[email protected]"},
Subject: "hi",
Body: `<h3>hi goio</h3><p>请点击下面的链接进行修改密码:http://www.baidu.com</p>`,
}
New(conf).Send(msg)
}
func TestMail_Send_gmail(t *testing.T) {
conf := Config{
Username: "[email protected]",
Password: "ljtgepqpmraiixeea",
Host: "smtp.gmail.com",
Port: 465,
TLS: true,
}
msg := Message{
Sender: "[email protected]",
SenderName: "测试者",
Receivers: []string{"[email protected]"},
Subject: "hi",
Body: `<h3>hi goio</h3><p>请点击下面的链接进行修改密码:http://www.baidu.com</p>`,
}
New(conf).Send(msg)
}
go-mail/message.go (761 B)
package gomail
import (
"bytes"
"fmt"
"strings"
)
type Message struct {
Sender string // 发送者
SenderName string // 发送者名称
Receivers []string // 接受者
Subject string // 主题
Body string // 内容
}
func (m *Message) Html() []byte {
var bf bytes.Buffer
bf.WriteString("To: " + strings.Join(m.Receivers, ",\r\n "))
bf.WriteString("\r\n")
if m.SenderName != "" {
bf.WriteString(fmt.Sprintf("From: \"%s\"<%s>", m.SenderName, m.Sender))
} else {
bf.WriteString("From: " + m.Sender)
}
bf.WriteString("\r\n")
bf.WriteString("Subject: " + m.Subject)
bf.WriteString("\r\n")
bf.WriteString("Content-Type:text/html;charset=utf-8")
bf.WriteString("\r\n\r\n")
bf.WriteString(m.Body)
return bf.Bytes()
}
go-mail/readme.md (30 B)
golang email
待测试Test
go-marketing/ReadMe.md (478 B)
Marketing API
Marketing API 是一套投放平台和变现平台功能的接口服务。
通过 Marketing API 可以帮助广告客户构建数据闭环,
客户可接入 Marketing API 将自有平台与小红书广告投放平台整合打通,并且实现广告投放过程中的各项功能。
包括推广任务的展示量、点击量、下载量、深度转化次数等,并且可以选择日期范围、添加过滤器,实现自定义推广数据报表。
go-marketing/go-admob/ReadMe.md (6.7 KiB)
Google admob data Api
文档地址
- https://developers.google.com/admob/api/release-notes?hl=zh-cn
- https://developers.google.com/admob/api/v1/how-tos/authorizing?hl=zh-cn
Admob API 授权
官方调试平台(授权 获取Token 访问)
https://developers.google.com/oauthplayground/
1 把 redirect_uri 改为自己的服务器地址
https://accounts.google.com/o/oauth2/v2/auth?redirect_uri=https%3A%2F%2Fbangbox.jidianle.cc&prompt=consent&response_type=code&client_id=&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fadmob.readonly+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fadmob.report&access_type=offline
跳转后选择权限 需要授权账号
2 授权后会返回 code
3 获取AccessToken 和 RefreshToken
4 需求
- 按产品平台(安卓/IOS)、国家(全部国家、美国、德国、荷兰、瑞典、英国、澳大利亚、加拿大、巴西、缅甸、伊朗)、天、天小时数据 广告类型和广告单元 POST https://admob.googleapis.com/v1/accounts/pub-4328354313035484/networkReport:generate
https://admob.googleapis.com/v1/accounts/pub-4328354313035484/networkReport:generate
curl -X POST \
https://admob.googleapis.com/v1/accounts/pub-XXXXXXXX/networkReport:generate \
-H "Content-Type: application/json" \
-H "Authorization: Bearer token" \
-H "X-Goog-AuthUser": "0" \
--data @- << EOF
{
"report_spec": {
"date_range": {
"start_date": {"year": 2024, "month": 4, "day": 1},
"end_date": {"year": 2024, "month": 4, "day": 2}
},
"dimensions": ["DATE"],
"metrics": ["CLICKS", "AD_REQUESTS", "IMPRESSIONS", "ESTIMATED_EARNINGS"],
"dimension_filters": [{"dimension": "COUNTRY", "matches_any": {"values": ["US"]}}],
"sort_conditions": [{"metric":"CLICKS", order: "DESCENDING"}],
"localization_settings": {"currency_code": "USD", "language_code": "en-US"}
}
}
EOF
[ {
"header" : {
"dateRange" : {
"startDate" : {
"year" : 2024,
"month" : 8,
"day" : 20
},
"endDate" : {
"year" : 2024,
"month" : 8,
"day" : 21
}
},
"localizationSettings" : {
"currencyCode" : "USD"
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~1283901700",
"displayLabel" : "JumpJumpVPN- Fast & Secure VPN"
},
"COUNTRY" : {
"value" : "AD"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "0"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "238"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~1283901700",
"displayLabel" : "JumpJumpVPN- Fast & Secure VPN"
},
"COUNTRY" : {
"value" : "AE"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "22"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "1427375"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~2152486667",
"displayLabel" : "VPN - biubiuVPN Fast & Secure"
},
"COUNTRY" : {
"value" : "AE"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "223"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "7850960"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~7713423190",
"displayLabel" : "biubiuVPN : VPN"
},
"COUNTRY" : {
"value" : "AE"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "71"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "4050666"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~8600583746",
"displayLabel" : "JumpJumpVPN- Fast & Secure VPN"
},
"COUNTRY" : {
"value" : "AE"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "259"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "6430629"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~1283901700",
"displayLabel" : "JumpJumpVPN- Fast & Secure VPN"
},
"COUNTRY" : {
"value" : "AF"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "0"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "3914"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~2152486667",
"displayLabel" : "VPN - biubiuVPN Fast & Secure"
},
"COUNTRY" : {
"value" : "AF"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "13"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "127095"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~7713423190",
"displayLabel" : "biubiuVPN : VPN"
},
"COUNTRY" : {
"value" : "AF"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "2"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "60361"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~8600583746",
"displayLabel" : "JumpJumpVPN- Fast & Secure VPN"
},
"COUNTRY" : {
"value" : "AF"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "8"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "67432"
}
}
}
}, {
"row" : {
"dimensionValues" : {
"DATE" : {
"value" : "20240820"
},
"APP" : {
"value" : "ca-app-pub-4328354313035484~2152486667",
"displayLabel" : "VPN - biubiuVPN Fast & Secure"
},
"COUNTRY" : {
"value" : "AL"
}
},
"metricValues" : {
"CLICKS" : {
"integerValue" : "0"
},
"ESTIMATED_EARNINGS" : {
"microsValue" : "341"
}
}
}
}, {
"footer" : {
"matchingRowCount" : "846"
}
} ]
go-marketing/go-admob/client.go (1.1 KiB)
package goadmob
import (
"context"
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoAdmob{}
// 可以一次初始化多个实例或者 多次调用初始化多个实例
func Init(ctx context.Context, configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("GoAdmob [" + name + "] already exists")
}
__clients[name], err = New(ctx, conf)
if err != nil {
return
}
}
return
}
func GetClient(names ...string) *GoAdmob {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoAdmob {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("GoAdmob").Error("no default GoAdmob client")
return nil
}
go-marketing/go-admob/config.go (608 B)
package goadmob
import gooauth "github.com/gif-gif/go.io/go-sso/go-oauth"
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
AccountId string `yaml:"AccountId" json:"accountId,optional"`
AuthConfig gooauth.Config
//AccessToken string `yaml:"AccessToken" json:"accessToken"`
//RefreshToken string `yaml:"RefreshToken" json:"refreshToken"`
//ClientId string `yaml:"ClientId" json:"clientId"`
//ClientSecret string `yaml:"ClientSecret" json:"clientSecret"`
//RedirectUrl string `yaml:"RedirectUrl" json:"redirectUrl"`
//State string `yaml:"State" json:"state"`
}
go-marketing/go-admob/goadmob.go (12.4 KiB)
package goadmob
import (
"context"
"errors"
gohttp "github.com/gif-gif/go.io/go-http"
golog "github.com/gif-gif/go.io/go-log"
gooauth "github.com/gif-gif/go.io/go-sso/go-oauth"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/admob/v1"
"google.golang.org/api/option"
"time"
)
const (
DefaultCurrencyCode = "USD"
DefaultLanguageCode = "en-US"
)
// 常用纬度
const (
FILTER_APP = "APP"
FILTER_COUNTRY = "COUNTRY"
FILTER_DATE = "DATE"
FILTER_PLATFORM = "PLATFORM"
FILTER_AD_UNIT = "AD_UNIT"
FILTER_FORMAT = "FORMAT"
FILTER_APP_VERSION_NAME = "APP_VERSION_NAME"
)
const (
AD_REQUESTS = "AD_REQUESTS"
CLICKS = "CLICKS"
ESTIMATED_EARNINGS = "ESTIMATED_EARNINGS"
IMPRESSIONS = "IMPRESSIONS"
IMPRESSION_CTR = "IMPRESSION_CTR"
IMPRESSION_RPM = "IMPRESSION_RPM"
MATCHED_REQUESTS = "MATCHED_REQUESTS"
MATCH_RATE = "MATCH_RATE"
SHOW_RATE = "SHOW_RATE"
)
var DefaultMetrics = []string{AD_REQUESTS, CLICKS, ESTIMATED_EARNINGS, IMPRESSIONS, IMPRESSION_CTR, IMPRESSION_RPM, MATCHED_REQUESTS, MATCH_RATE, SHOW_RATE}
type ReportReq struct {
Dimensions []string //查询维度列表
AdUnits []string //广告位
Countries []string //国家
AdFormats []string //原生、横幅、插屏、开屏、激励视频
Platforms []string //应用的移动操作系统平台(例如“Android”或“iOS”)。
AppVersionNames []string //对于 Android,应用版本名称可以在 PackageInfo 中的 versionName 中找到。对于 iOS,可以在 CFBundleShortVersionString 中找到应用版本名称。警告:该维度与 ESTIMATED_EARNINGS 和 OBSERVED_ECPM 指标不兼容。
MaxReportRows int64 //最大返回数量
Metrics []string //查询字段、注意:这里的查询字段在不同的纬度下有互斥情况
StartDate admob.Date
EndDate admob.Date
CurrencyCode string //default currency USD
LanguageCode string //default language en-US
RetryWaitTime time.Duration
RetryCount int
//Date Month Week
}
type ResponseItem struct {
AppVersionName string `json:"app_version_name"`
Platform string `json:"platform"`
Format string `json:"format"`
Date string `json:"date"`
AdUnit string `json:"ad_unit"`
Country string `json:"country"`
AdRequest int64 `json:"ad_requests"`
Clicks int64 `json:"clicks"`
Earnings int64 `json:"earnings"` // 美分*10000. 这里如果 返回美分会损失精度
Impressions int64 `json:"impressions"`
ImpressionCtr float64 `json:"impression_ctr"`
ImpressionRpm float64 `json:"Impression_rpm"` //美分
MatchedRequests int64 `json:"matched_requests"`
MatchRate float64 `json:"match_rate"`
ShowRate float64 `json:"show_rate"`
}
type ReportResponse struct {
Items []*ResponseItem
NextPageToken string
}
// accessToken 会在60分钟后过期
type GoAdmob struct {
ctx context.Context
Config Config
GoAuth *gooauth.GoOAuth
AdmobService *admob.Service
RequestTimeout int64 //default request timeout 30s
}
// 每次调用时都需要调用这个方法
func New(ctx context.Context, config Config) (*GoAdmob, error) {
if len(config.AuthConfig.Scopes) <= 0 {
config.AuthConfig.Scopes = []string{"https://www.googleapis.com/auth/admob.readonly", "https://www.googleapis.com/auth/admob.report"}
}
if config.AuthConfig.Endpoint == nil {
config.AuthConfig.Endpoint = &gooauth.Endpoint{
TokenURL: google.Endpoint.TokenURL,
AuthURL: google.Endpoint.AuthURL,
DeviceAuthURL: google.Endpoint.DeviceAuthURL,
AuthStyle: google.Endpoint.AuthStyle,
}
}
err := gooauth.Init(config.AuthConfig)
if err != nil {
return nil, err
}
o := &GoAdmob{
ctx: ctx,
Config: config,
GoAuth: gooauth.GetClient(config.AuthConfig.Name),
}
return o, nil
}
// RefreshToken and AdmobService admon token 有效期为60分钟,所有每次请求数据刷新下token
func (o *GoAdmob) Refresh() error {
err := o.RefreshToken()
if err != nil {
return err
}
admobService, err := admob.NewService(o.ctx, option.WithTokenSource(o.GoAuth.TokenSource(o.ctx)))
if err != nil {
return err
}
o.AdmobService = admobService
return nil
}
// 授权admobURL OAuth2.0
//
// 1、https://developers.google.com/admob/api/v1/getting-started?hl=zh-cn 创建一个新的授权客户端 获取client_id 和 client_secret
//
// 2、浏览器中执行 AuthUrl 方法获取授权code
//
// 3、获取code后执行 Exchange 方法获取token
func (c *GoAdmob) AuthUrl() string {
return c.GoAuth.AuthUrl("")
}
// 获取token
func (c *GoAdmob) Exchange(ctx context.Context, authorizationCode string) (*oauth2.Token, error) {
token, err := c.GoAuth.Exchange(ctx, authorizationCode)
if err != nil {
golog.WithTag("goadmob").Error("token:" + err.Error())
return nil, err
}
return token, nil
}
// 刷新token
func (c *GoAdmob) RefreshToken() error {
_, err := c.GoAuth.RefreshToken(c.ctx)
if err != nil {
golog.WithTag("goadmob").Error("token:" + err.Error())
return err
}
return nil
}
// 获取广告报表信息 这个接口不支持查询维度,返回接口有bug
//func (c *GoAdmob) GetReport() (*admob.GenerateNetworkReportResponse, error) {
// res, err := c.AdmobService.Accounts.NetworkReport.Generate("accounts/"+c.Config.AccountId, &admob.GenerateNetworkReportRequest{
// ReportSpec: &admob.NetworkReportSpec{
// DateRange: &admob.DateRange{
// EndDate: &admob.Date{
// Day: 21,
// Month: 8,
// Year: 2024,
// },
// StartDate: &admob.Date{
// Day: 20,
// Month: 8,
// Year: 2024,
// },
// },
// Dimensions: []string{"DATE", "APP", "COUNTRY"},
// //DimensionFilters: []*admob.NetworkReportSpecDimensionFilter{
// // {
// // Dimension: "COUNTRY",
// // MatchesAny: &admob.StringList{
// // Values: []string{"US"},
// // },
// // },
// //},
// MaxReportRows: 10,
// Metrics: []string{"CLICKS", "ESTIMATED_EARNINGS"},
// },
// }).Do()
// return res, err
//}
// dimensions 指定查询维度,如:[]string{"DATE", "APP", "COUNTRY"}
//
// SELECT DATE, APP, COUNTRY, CLICKS, ESTIMATED_EARNINGS
// FROM NETWORK_REPORT
// WHERE DATE >= '2021-09-01' AND DATE <= '2021-09-30'
//
// AND COUNTRY IN ('US', 'CN')
//
// GROUP BY DATE, APP, COUNTRY
// ORDER BY APP ASC, CLICKS DESC;
func (c *GoAdmob) GetReport(req *ReportReq) (*ReportResponse, error) {
if req.MaxReportRows == 0 {
return nil, errors.New("MaxReportRows is empty")
}
url := "/v1/accounts/" + c.Config.AccountId + "/networkReport:generate"
dimensionFilters := []*admob.NetworkReportSpecDimensionFilter{}
if len(req.AdUnits) > 0 {
dimensionFilters = append(dimensionFilters, &admob.NetworkReportSpecDimensionFilter{
Dimension: FILTER_AD_UNIT,
MatchesAny: &admob.StringList{
Values: req.AdUnits,
},
})
}
if len(req.AdFormats) > 0 {
dimensionFilters = append(dimensionFilters, &admob.NetworkReportSpecDimensionFilter{
Dimension: FILTER_FORMAT,
MatchesAny: &admob.StringList{
Values: req.AdFormats,
},
})
}
if len(req.Platforms) > 0 {
dimensionFilters = append(dimensionFilters, &admob.NetworkReportSpecDimensionFilter{
Dimension: FILTER_PLATFORM,
MatchesAny: &admob.StringList{
Values: req.Platforms,
},
})
}
if len(req.AppVersionNames) > 0 {
dimensionFilters = append(dimensionFilters, &admob.NetworkReportSpecDimensionFilter{
Dimension: FILTER_APP_VERSION_NAME,
MatchesAny: &admob.StringList{
Values: req.AppVersionNames,
},
})
}
if len(req.Countries) > 0 {
dimensionFilters = append(dimensionFilters, &admob.NetworkReportSpecDimensionFilter{
Dimension: FILTER_COUNTRY,
MatchesAny: &admob.StringList{
Values: req.Countries,
},
})
}
if len(req.Metrics) == 0 {
req.Metrics = DefaultMetrics
}
if req.CurrencyCode == "" {
req.CurrencyCode = DefaultCurrencyCode
}
if req.LanguageCode == "" {
req.LanguageCode = DefaultLanguageCode
}
params := &admob.GenerateNetworkReportRequest{
ReportSpec: &admob.NetworkReportSpec{
DateRange: &admob.DateRange{
EndDate: &req.EndDate,
StartDate: &req.StartDate,
},
Dimensions: req.Dimensions, //[]string{"DATE", "APP", "COUNTRY"}, //group by
DimensionFilters: dimensionFilters,
MaxReportRows: req.MaxReportRows,
Metrics: req.Metrics,
LocalizationSettings: &admob.LocalizationSettings{
CurrencyCode: req.CurrencyCode,
LanguageCode: req.LanguageCode,
},
},
}
dataReq := &gohttp.Request{
Url: url,
RetryCount: req.RetryCount,
RetryWaitTime: req.RetryWaitTime,
Timeout: time.Second * 30,
Body: params,
}
gh := &gohttp.GoHttp[[]*admob.GenerateNetworkReportResponse]{
Request: dataReq,
BaseUrl: "https://admob.googleapis.com",
Headers: map[string]string{
"X-Google-AuthUser": "0",
"Authorization": "Bearer " + c.GoAuth.GetAccessToken(),
},
}
res, err := gh.HttpPostJson(c.ctx)
if err != nil {
return nil, err
}
list := []*ResponseItem{}
for _, response := range *res {
item := ResponseItem{}
row := response.Row
if row != nil {
//维度
for fieldName, value := range row.DimensionValues {
switch fieldName {
case FILTER_DATE:
item.Date = value.Value
break
case FILTER_APP:
item.AdUnit = value.Value
break
case FILTER_COUNTRY:
item.Country = value.Value
break
case FILTER_APP_VERSION_NAME:
item.AppVersionName = value.Value
break
case FILTER_PLATFORM:
item.Platform = value.Value
break
case FILTER_AD_UNIT:
item.AdUnit = value.Value
break
case FILTER_FORMAT:
item.Format = value.Value
break
}
}
//指标
for fieldName, value := range row.MetricValues {
switch fieldName {
case AD_REQUESTS:
item.AdRequest = value.IntegerValue
break
case CLICKS:
item.Clicks = value.IntegerValue
break
case ESTIMATED_EARNINGS:
item.Earnings = value.MicrosValue
break
case IMPRESSIONS:
item.Impressions = value.IntegerValue
break
case IMPRESSION_CTR:
item.ImpressionCtr = value.DoubleValue
break
case IMPRESSION_RPM:
item.ImpressionRpm = value.DoubleValue
break
case MATCHED_REQUESTS:
item.MatchedRequests = value.IntegerValue
break
case MATCH_RATE:
item.MatchRate = value.DoubleValue
break
case SHOW_RATE:
item.ShowRate = value.DoubleValue
break
}
}
list = append(list, &item)
}
}
response := &ReportResponse{
Items: list,
NextPageToken: "",
}
return response, nil
}
// 获取账号下所有APP信息
func (c *GoAdmob) GetApps(pageSize int64, nextPageToken string) (*admob.ListAppsResponse, error) {
if c.Config.AccountId == "" {
return nil, errors.New("GetApps accountId is empty")
}
if c.AdmobService == nil {
return nil, errors.New("GetApps AdmobService is empty")
}
if c.AdmobService.Accounts == nil {
return nil, errors.New("GetApps AdmobService.Accounts is empty")
}
if c.AdmobService.Accounts.Apps == nil {
return nil, errors.New("GetApps sAdmobService.Accounts.Apps is empty")
}
res, err := c.AdmobService.Accounts.Apps.List("accounts/" + c.Config.AccountId).PageSize(pageSize).PageToken(nextPageToken).Do()
if err != nil {
return nil, err
}
return res, nil
}
// 获取当前appId下所有的广告信息
func (c *GoAdmob) GetAdUnits(pageSize int64, nextPageToken string) (*admob.ListAdUnitsResponse, error) {
if c.Config.AccountId == "" {
return nil, errors.New("accountId is empty")
}
if c.AdmobService == nil {
return nil, errors.New("GetAdUnits AdmobService is empty")
}
if c.AdmobService.Accounts == nil {
return nil, errors.New("GetAdUnits AdmobService.Accounts is empty")
}
if c.AdmobService.Accounts.AdUnits == nil {
return nil, errors.New("GetAdUnits sAdmobService.Accounts.AdUnits is empty")
}
res, err := c.AdmobService.Accounts.AdUnits.List("accounts/" + c.Config.AccountId).PageSize(pageSize).PageToken(nextPageToken).Do()
if err != nil {
return nil, err
}
return res, nil
}
func (c *GoAdmob) GetAccountInfo() (*admob.PublisherAccount, error) {
if c.Config.AccountId == "" {
return nil, errors.New("accountId is empty")
}
if c.AdmobService == nil {
return nil, errors.New("GetAccountInfo AdmobService is empty")
}
if c.AdmobService.Accounts == nil {
return nil, errors.New("GetAccountInfo AdmobService.Accounts is empty")
}
return c.AdmobService.Accounts.Get("accounts/" + c.Config.AccountId).Do()
}
go-marketing/go-admob/goadmob_test.go (4.1 KiB)
package goadmob
import (
"context"
gocontext "github.com/gif-gif/go.io/go-context"
"github.com/gif-gif/go.io/go-db/gogorm"
golog "github.com/gif-gif/go.io/go-log"
gooauth "github.com/gif-gif/go.io/go-sso/go-oauth"
"google.golang.org/api/admob/v1"
"testing"
"time"
)
func init() {
//gogorm.Init(gogorm.Config{
// DataSource: "root:111111@tcp(127.0.0.1)/admob?charset=utf8mb4&parseTime=True&loc=Local",
//})
}
func TestAdmobApps(t *testing.T) {
err := Default().Refresh()
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
apps, err := Default().GetApps(100, "")
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
type AdAccount struct {
Id int64
AccountId string
Channel string
Status int64
CreateTime int64
UpdateTime int64
}
adAccount := AdAccount{}
db := gogorm.Default().DB
err = db.Table("ad_account").Select("id,account_id,channel,status,create_time,update_time").First(&adAccount).Error
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
for _, app := range apps.Apps {
err = db.Table("ad_app").Exec("insert into ad_app(app_code,platform,title,ad_account_id,app_pub_id,app_store_id,status,create_time,update_time)values(?,?,?,?,?,?,?,?,?)", app.LinkedAppInfo.DisplayName, app.Platform, app.LinkedAppInfo.DisplayName, adAccount.Id, app.AppId, app.LinkedAppInfo.AppStoreId, 1, time.Now().Unix(), time.Now().Unix()).Error
if err != nil {
golog.WithTag("godb").Error(err.Error())
return
}
golog.WithTag("admob").WithField("appId", app.AppId).Info("OK")
}
<-gocontext.WithCancel().Done()
}
type AdApp struct {
Id int64
AppPubId string
}
func TestAdmobAdUnits(t *testing.T) {
var adAppMap = make(map[string]int64)
db := gogorm.Default().DB
adApps := []AdApp{}
err := db.Table("ad_app").Select("id,app_pub_id").Scan(&adApps).Error
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
for _, app := range adApps {
adAppMap[app.AppPubId] = app.Id
}
err = Default().Refresh()
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
apps, err := Default().GetAdUnits(1000, "")
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
for _, app := range apps.AdUnits {
appId := adAppMap[app.AppId]
err = db.Table("ad_info").Exec("insert into ad_info(ad_app_id,title,ad_type,ad_unit,status,create_time,update_time)values(?,?,?,?,?,?,?)", appId, app.DisplayName, app.AdFormat, app.AdUnitId, 1, time.Now().Unix(), time.Now().Unix()).Error
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
golog.WithTag("admob").WithField("appId", app.AdUnitId).Info("OK")
}
<-gocontext.WithCancel().Done()
}
func TestAdmobReport(t *testing.T) {
var adAppMap = make(map[string]int64)
db := gogorm.Default().DB
adApps := []AdApp{}
err := db.Table("ad_app").Select("id,app_pub_id").Scan(&adApps).Error
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
for _, app := range adApps {
adAppMap[app.AppPubId] = app.Id
}
err = Default().Refresh()
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
req := &ReportReq{
MaxReportRows: 100,
Dimensions: []string{"DATE", "APP", "COUNTRY"},
AdUnits: []string{"ca-app-pub-4328354313035484/2826310677"},
Metrics: []string{"AD_REQUESTS", "CLICKS", "ESTIMATED_EARNINGS", "IMPRESSIONS", "IMPRESSION_CTR", "IMPRESSION_RPM", "MATCHED_REQUESTS", "MATCH_RATE", "SHOW_RATE"},
EndDate: admob.Date{
Day: 21,
Month: 8,
Year: 2024,
},
StartDate: admob.Date{
Day: 20,
Month: 8,
Year: 2024,
},
}
res, err := Default().GetReport(req)
if err != nil {
golog.WithTag("goadmob").Error(err.Error())
return
}
golog.WithTag("admob").Info(res)
<-gocontext.WithCancel().Done()
}
func TestAdmobAuthUrl(t *testing.T) {
Init(context.Background(), Config{
Name: "admob",
AccountId: "123",
AuthConfig: gooauth.Config{
ClientSecret: "secret",
RedirectURL: "https://test.com",
},
})
url := Default().AuthUrl()
golog.WithTag("admob").Info(url)
}
func TestAdmobAuth(t *testing.T) {
}
go-marketing/go-amazon/ip-range/ip_range.go (1.6 KiB)
package amazon_iprange
import (
"context"
gohttp "github.com/gif-gif/go.io/go-http"
)
var (
ipRange *IpRange
IpRangeUrl = "https://ip-ranges.amazonaws.com/ip-ranges.json"
)
type AmazonIp4 struct {
IpPrefix string `json:"ip_prefix"`
Region string `json:"region"`
Service string `json:"service"`
NetworkBorderGroup string `json:"network_border_group"`
}
type AmazonIpV6 struct {
Ipv6Prefix string `json:"ipv6_prefix"`
Region string `json:"region"`
Service string `json:"service"`
NetworkBorderGroup string `json:"network_border_group"`
}
type IpRange struct {
SyncToken string `json:"syncToken"`
CreateDate string `json:"createDate"`
Prefixes []AmazonIp4 `json:"prefixes"`
Ipv6Prefixes []AmazonIpV6 `json:"ipv6_prefixes"`
}
func LoadRangeIps() (*IpRange, error) {
req := &gohttp.Request{
Url: IpRangeUrl,
Method: gohttp.GET,
Headers: map[string]string{"User-Agent": "github.com/gif-gif/go.io"},
}
gh := gohttp.GoHttp[IpRange]{
Request: req,
}
res, err := gh.HttpGet(context.Background())
if err != nil {
return nil, err
}
return res, nil
}
// 缓存ipRange信息
// GetCacheIpRanges 方法获取
func LoadRangeIpsAndCache() error {
req := &gohttp.Request{
Url: IpRangeUrl,
Method: gohttp.GET,
Headers: map[string]string{"User-Agent": "github.com/gif-gif/go.io"},
}
gh := gohttp.GoHttp[IpRange]{
Request: req,
}
res, err := gh.HttpGet(context.Background())
if err != nil {
return err
}
ipRange = res
return nil
}
func GetCacheIpRanges() *IpRange {
return ipRange
}
go-marketing/go-baidu/ReadMe.md (22 B)
Baidu Marketing Api
go-marketing/go-googleads/ReadMe.md (27 B)
Google ads Marketing Api
go-marketing/go-meta/ReadMe.md (894 B)
Meta(Facebook) Marketing Api
1 meta marking api
- https://developers.facebook.com/docs/marketing-apis/
授权
- https://developers.facebook.com/docs/marketing-api/overview/authorization
2 meta audience api
- https://developers.facebook.com/docs/audience-network/
- https://developers.facebook.com/docs/audience-network/optimization/report-api
授权
文档:
https://developers.facebook.com/docs/audience-network/optimization/apis/FB-login-Reporting-API#getting-permissions
授权页面:
https://developers.facebook.com/docs/audience-network/optimization/apis/FB-login-Reporting-API#getting-permissions
- https://developers.facebook.com/tools/explorer 短期口令(2个小时)
- https://developers.facebook.com/tools/debug/accesstoken 延长口令(如果不访问60天,访问时自动延长)
- 2 meta 开发工具
- https://developers.facebook.com/tools/
go-marketing/go-meta/audience.go (4.2 KiB)
package gometa
import (
"context"
"fmt"
gohttp "github.com/gif-gif/go.io/go-http"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gogf/gf/util/gconv"
"github.com/google/go-querystring/query"
"strings"
"time"
)
type AudienceResponseItem struct {
AppVersionName string `json:"app_version_name"`
Platform string `json:"platform"`
Format string `json:"format"`
Date string `json:"date"`
Hour int64 `json:"hour"`
AdUnit string `json:"ad_unit"`
Country string `json:"country"`
AdRequest int64 `json:"ad_requests"`
Clicks int64 `json:"clicks"`
Earnings int64 `json:"earnings"` // 1美元=1000000微分,美分*10000. 这里如果 返回美分会损失精度
Impressions int64 `json:"impressions"`
ImpressionCtr float64 `json:"impression_ctr"`
ImpressionRpm float64 `json:"Impression_rpm"` //美元
MatchedRequests int64 `json:"matched_requests"`
MatchRate float64 `json:"match_rate"`
ShowRate float64 `json:"show_rate"`
}
type AudienceReportResponse struct {
Items []*AudienceResponseItem
NextPageToken string
}
// <ID> 是您的 Meta 企业编号、资产编号或应用编号
func (m *GoMeta) GetMetaAudienceReport(req *AudienceDataRequest, ID string) (*AudienceDataResponse, error) {
if req.Limit > limitMax {
return nil, fmt.Errorf("最大查询不能超过:" + gconv.String(limitMax) + "条")
}
api := m.Config.currentVersionBaseApi + ApiAdNetworkAnalytics
api = fmt.Sprintf(api, ID)
params, _ := query.Values(req)
request := &gohttp.Request{
Url: api,
ParamsValues: params,
}
gh := gohttp.GoHttp[AudienceDataResponse]{Request: request}
result, err := gh.HttpGet(context.Background())
if err != nil {
return nil, err
}
return result, nil
}
func (m *GoMeta) GetAudienceReport(req *AudienceDataRequest, ID string) (*AudienceReportResponse, error) {
rst, err := m.GetMetaAudienceReport(req, ID)
if err != nil {
return nil, err
}
res := AudienceReportResponse{}
rowsData := make(map[string]map[string]string) //一行数据, 列转行
for _, item := range rst.Data {
for _, result := range item.Results {
key := result.Time
for _, breakdown := range result.Breakdowns {
key += "-" + breakdown.Key + ":" + breakdown.Value + ""
}
rowData := make(map[string]string)
if rowsData[key] == nil {
rowsData[key] = rowData
} else {
rowData = rowsData[key]
}
rowData[result.Metric] = result.Value
rowsData[key] = rowData
}
}
for rowsKey, value := range rowsData {
vo := AudienceResponseItem{}
keys := strings.Split(rowsKey, "-")
for i, key := range keys {
if i == 0 {
tt, err := goutils.ConvertToGMTTime(key)
if err != nil {
fmt.Println(err)
return nil, err
}
// 将时间对象转换为 UTC 时区
utcTime := tt.UTC()
gmtTimeStr := utcTime.Format(time.DateOnly)
hour := utcTime.Hour()
vo.Date = gmtTimeStr
vo.Hour = int64(hour)
} else {
kk := strings.Split(key, ":")
switch kk[0] {
case BREAKDOWN_COUNTRY:
vo.Country = kk[1]
break
case BREAKDOWN_PLACEMENT:
vo.AdUnit = kk[1]
break
case BREAKDOWN_PLATFORM:
vo.Platform = kk[1]
break
}
}
}
for metric, v := range value {
switch metric {
case Metrics_AD_NETWORK_REVENUE:
vo.Earnings = gconv.Int64(gconv.Float64(v) * DOLLAR_UNIT) //1美元=1000000微分, market api 统一返回格式
break
case Metrics_AD_NETWORK_CPM:
vo.ImpressionRpm = gconv.Float64(v)
break
case Metrics_AD_NETWORK_IMP:
vo.Impressions = gconv.Int64(v)
break
case Metrics_AD_NETWORK_REQUEST:
vo.AdRequest = gconv.Int64(v)
break
case Metrics_AD_NETWORK_SHOW_RATE:
vo.ShowRate = gconv.Float64(v)
break
case Metrics_AD_NETWORK_CTR:
vo.ImpressionCtr = gconv.Float64(v)
break
case Metrics_AD_NETWORK_CLICK:
vo.Clicks = gconv.Int64(v)
break
case Metrics_AD_NETWORK_FILL_RATE:
vo.MatchRate = gconv.Float64(v)
break
case Metrics_AD_NETWORK_FILLED_REQUEST:
vo.MatchedRequests = gconv.Int64(v)
break
}
}
res.Items = append(res.Items, &vo)
}
res.NextPageToken = rst.Paging.Cursors.After
return &res, nil
}
go-marketing/go-meta/audience_types.go (6.3 KiB)
package gometa
// API
const (
ApiAdNetworkAnalytics = "/%s/adnetworkanalytics" // 变现广告网络分析
)
// “每分钟最多执行 250 次查询”
const limitMax = 500 //返回的行数。限制:同步请求的数量上限为 2,000。
// 常用纬度 group by 数据
// breakdowns breakdowns=['breakdown_1', 'breakdown_2',...]
const (
BREAKDOWN_AD_SPACE = "ad_space" //按广告专区细分
BREAKDOWN_COUNTRY = "country" //按国家/地区细分
BREAKDOWN_DELIVERY_METHOD = "delivery_method" //如果指标来源于通过 Audience Network 竞价投放的广告,则按 standard 或 bidding 细分。仅适用于使用变现管理工具的发行商。
BREAKDOWN_fAIL_REASON = "fail_reason" //仅适用于 fb_ad_network_no_fill 和 fb_ad_network_no_bid 指标。
BREAKDOWN_PLACEMENT = "placement" //按版位编号细分。不能与 placement_name 一起使用。
BREAKDOWN_PLACEMENT_NAME = "placement_name" //按版位编号和名称细分。不能与 placement 一起使用。
BREAKDOWN_PLATFORM = "platform" //按平台细分。可以是 ios、android、mobile_web 或 instant_games。
BREAKDOWN_PROPERTY = "property" //按资产编号细分
)
// filters
const (
FILTER_COUNTRY = "country" // country 以逗号分隔的双字母国家/地区缩写的清单
FILTER_PLACEMENT = "placement" // placement 版位编号。限制:如果展示次数不足,值是 REDACTED。
FILTER_DELIVERY_METHOD = "delivery_method" // delivery_method standard 或 bidding
FILTER_PLATFORM = "platform" // 可以是 ios(移动应用)、android(移动应用)、mobile_web 或 instant_games。
)
// aggregation_period=hour|day|total
const (
AGGREGATION_PERIOD_HOUR = "hour"
AGGREGATION_PERIOD_DAY = "day"
AGGREGATION_PERIOD_TOTAL = "total"
)
// 指标
const (
Metrics_AD_NETWORK_BIDDING_BID_RATE = "fb_ad_network_bidding_bid_rate" // 竞价响应率
Metrics_AD_NETWORK_BIDDING_REQUEST = "fb_ad_network_bidding_request" // 竞价请求数量
Metrics_AD_NETWORK_BIDDING_RESPONSE = "fb_ad_network_bidding_response" // 竞价响应数量
Metrics_AD_NETWORK_BIDDING_WIN_RATE = "fb_ad_network_bidding_win_rate" // 竞价工具赢得竞拍的比率
Metrics_AD_NETWORK_CLICK = "fb_ad_network_click" // 点击量
Metrics_AD_NETWORK_CPM = "fb_ad_network_cpm" // 有效千次展示费用 (eCPM)
Metrics_AD_NETWORK_CTR = "fb_ad_network_ctr" // 预估点击率
Metrics_AD_NETWORK_FILL_RATE = "fb_ad_network_fill_rate" // 广告请求填充率
Metrics_AD_NETWORK_FILLED_REQUEST = "fb_ad_network_filled_request" // 填充的广告请求数量
Metrics_AD_NETWORK_IMP = "fb_ad_network_imp" // 展示次数
Metrics_AD_NETWORK_NO_BID = "fb_ad_network_no_bid" // 无响应竞价主因数量 仅适用于用作单个指标 fail_reason 细分条件的情况
Metrics_AD_NETWORK_NO_FILL = "fb_ad_network_no_fill" // 无填充主因数量仅适用于用作单个指标 fail_reason 细分条件的情况
Metrics_AD_NETWORK_REQUEST = "fb_ad_network_request" // 广告请求数量
Metrics_AD_NETWORK_REVENUE = "fb_ad_network_revenue" // 预估收入
Metrics_AD_NETWORK_SHOW_RATE = "fb_ad_network_show_rate" // 展示数除以填充请求数
)
var DefaultAudienceMetrics = []string{
Metrics_AD_NETWORK_IMP,
Metrics_AD_NETWORK_REQUEST,
Metrics_AD_NETWORK_CLICK,
Metrics_AD_NETWORK_CPM,
Metrics_AD_NETWORK_CTR,
Metrics_AD_NETWORK_REVENUE,
Metrics_AD_NETWORK_FILL_RATE,
Metrics_AD_NETWORK_SHOW_RATE,
Metrics_AD_NETWORK_BIDDING_REQUEST,
Metrics_AD_NETWORK_BIDDING_RESPONSE,
Metrics_AD_NETWORK_BIDDING_WIN_RATE,
Metrics_AD_NETWORK_BIDDING_BID_RATE,
Metrics_AD_NETWORK_NO_FILL,
Metrics_AD_NETWORK_NO_BID,
}
// ResponseData 数据
type AudienceData struct {
QueryId string `json:"query_id"`
Results []struct {
Time string `json:"time"`
Metric string `json:"metric"`
Breakdowns []struct {
Key string `json:"key"`
Value string `json:"value"`
} `json:"breakdowns"`
Value string `json:"value"`
} `json:"results"`
OmittedResults []struct {
Time string `json:"time"`
Metric string `json:"metric"`
Breakdowns []struct {
Key string `json:"key"`
Value string `json:"value"`
} `json:"breakdowns"`
} `json:"omitted_results"`
}
type AudienceFilter struct {
Field string `json:"field"`
Operator string `json:"operator"`
Values []string `json:"values"`
}
// https://developers.facebook.com/docs/audience-network/optimization/report-api/guide-v2/
// filters=[{'field':'country', 'operator':'in', 'values':['US', 'JP']}]
type AudienceDataRequest struct {
AggregationPeriod string `url:"aggregation_period"` //aggregation_period=hour|day|total 按 day(默认)、hour 或 total 汇总结果。限制:如要按小时汇总结果,您必须使用 since 和 until 查询至少 2 天内的结果。
//Since 限制:
//如要使用 Unix 时间戳,您的查询范围必须至少为 1 小时。
//在同步请求中,您的请求范围最多为 8 天。
//数据只会保留 540 天。如要请求的数据时间范围超过 $currentDate - 539 days,则系统不会返回更多数据。
Since string `url:"since"` //since=YYYY-MM-DD 或 since=1548880485 查询的开始限制(始终包含边界值)。如果未添加此参数,默认为过去 7 天。
Until string `url:"until"` //until=YYYY-MM-DD 或 until=1548880485+86400 查询的结束限制(默认不包含边界值,如果查询的汇总数据精确到小时,则包含边界值)
Filter []AudienceFilter `url:"filters"`
Breakdowns []string `url:"breakdowns"`
Metrics []string `url:"metrics"`
Limit int64 `url:"limit"` //返回的行数。限制:同步请求的数量上限为 2,000。并限定每个指标获取 多少个响应
OrderingColumn string `url:"ordering_column"` //time|value,默认值为 time。
OrderingType string `url:"ordering_type"` //ascending|descending ,默认值为 descending。升序或降序。
After string `url:"after"` //下一页游标。
}
type AudienceDataResponse struct {
BaseResponse[[]AudienceData]
}
go-marketing/go-meta/client.go (1.1 KiB)
package gometa
import (
"context"
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoMeta{}
// 可以一次初始化多个实例或者 多次调用初始化多个实例
func Init(ctx context.Context, configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("GoMeta [" + name + "] already exists")
}
__clients[name] = New(conf)
}
return
}
func GetClient(names ...string) *GoMeta {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoMeta {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("GoMeta").Error("no default GoMeta client")
return nil
}
go-marketing/go-meta/gometa.go (3.0 KiB)
package gometa
import (
"context"
gohttp "github.com/gif-gif/go.io/go-http"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/google/go-querystring/query"
)
// 1美元=1000000微分
const (
DOLLAR_UNIT = 1000000
)
type Paging struct {
Limit int `json:"limit,optional"`
Before string `json:"before,optional"`
After string `json:"after,optional"`
}
// meta 通用返回数据结构
type BaseResponse[T any] struct {
Paging struct {
Cursors struct {
Before string `json:"before,optional"`
After string `json:"after,optional"`
} `json:"cursors,optional"`
Next string `json:"next,optional"`
} `json:"paging,optional"`
Data T `json:"data,optional"`
}
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
//请求参数
ApiVersion string `yaml:"ApiVersion" json:"apiVersion,optional"`
AccessToken string `yaml:"AccessToken" json:"accessToken"`
// 授权参数
ClientId string `yaml:"ClientId" json:"clientId"`
ClientSecret string `yaml:"ClientSecret" json:"clientSecret"`
RedirectUri string `yaml:"RedirectUri" json:"redirectUri"`
//基础API
baseApi string
currentVersionBaseApi string
}
type GoMeta struct {
Config Config
}
func New(config Config) *GoMeta {
mm := &GoMeta{
Config: config,
}
if mm.Config.ApiVersion == "" {
mm.Config.ApiVersion = "v21.0"
}
if mm.Config.baseApi == "" {
mm.Config.baseApi = "https://graph.facebook.com"
}
mm.Config.currentVersionBaseApi = mm.Config.baseApi + "/" + mm.Config.ApiVersion
return mm
}
// 刷新token接口
func (m *GoMeta) RefreshToken() (*TokenResponse, error) {
req := &ApiRefreshTokenRequest{
ClientId: m.Config.ClientId,
ClientSecret: m.Config.ClientSecret,
GrantType: "fb_exchange_token",
FbExchangeToken: m.Config.AccessToken,
}
api := m.Config.currentVersionBaseApi + ApiRefreshToken
params, _ := query.Values(req)
request := &gohttp.Request{
Url: api,
ParamsValues: params,
}
gh := gohttp.GoHttp[TokenResponse]{
Request: request,
}
result, err := gh.HttpGet(context.Background())
if err != nil {
return nil, err
}
return result, nil
}
// 获取token
func (c *GoMeta) Exchange(authorizationCode string) (*TokenResponse, error) {
req := &ApiAccessTokenRequest{
ClientId: c.Config.ClientId,
ClientSecret: c.Config.ClientSecret,
Code: authorizationCode,
RedirectUri: c.Config.RedirectUri,
}
api := c.Config.currentVersionBaseApi + ApiRefreshToken
params, _ := query.Values(req)
request := &gohttp.Request{
Url: api,
ParamsValues: params,
}
gh := gohttp.GoHttp[TokenResponse]{
Request: request,
}
result, err := gh.HttpGet(context.Background())
if err != nil {
return nil, err
}
return result, nil
}
// 授权URL
//
// DOC: https://developers.facebook.com/docs/marketing-api/overview/authorization
func (c *GoMeta) AuthUrl(scope string) string {
return c.Config.currentVersionBaseApi + "/dialog/oauth?client_id=" + c.Config.ClientId + "&redirect_uri=" + goutils.UrlEncode(c.Config.RedirectUri) + "&scope=" + goutils.UrlEncode(scope)
}
go-marketing/go-meta/market.go (4.2 KiB)
package gometa
import (
"context"
"fmt"
gohttp "github.com/gif-gif/go.io/go-http"
"github.com/gogf/gf/util/gconv"
"github.com/google/go-querystring/query"
"time"
)
func (m *GoMeta) handleRequest(req *RequestData) *RequestData {
req.TimeRange = "{'since':'" + req.DateStart + "','until':'" + req.DateStop + "'}"
req.DateStart = ""
req.DateStop = ""
if req.Timezone == "" {
//req.Timezone = "Asia/Shanghai"
}
if req.TimeIncrement <= 0 { //默认以天为单位
req.TimeIncrement = 1 //以天为单位返回
}
return req
}
func (m *GoMeta) DecryptEcpms(appId string, encryptedEcpms []string) (*EncryptedEcpmRes, error) {
api := m.Config.baseApi + "/" + appId + "/aggregate_revenue"
req := EncryptedEcpmReq{
AccessToken: m.Config.AccessToken,
Ecpms: encryptedEcpms,
RequestId: gconv.String(time.Now().UnixNano()),
SyncApi: true,
}
request := &gohttp.Request{
Url: api,
Body: req,
}
gh := gohttp.GoHttp[EncryptedEcpmRes]{Request: request}
result, err := gh.HttpPostJson(context.Background())
if err != nil {
return nil, err
}
return result, nil
}
// 某个商户下所有账号信息 账号余额,状态等等
func (m *GoMeta) GetMarketAccountsByBusinessId(businessId string, pageSize int) (*AccountResponse, error) {
if pageSize == 0 {
pageSize = 10000
}
api := m.Config.currentVersionBaseApi + ApiAccount
api = fmt.Sprintf(api, businessId)
req := &RequestData{
AccessToken: m.Config.AccessToken,
Limit: pageSize,
Fields: accountFields,
}
req = m.handleRequest(req)
params, _ := query.Values(req)
request := &gohttp.Request{
Url: api,
ParamsValues: params,
}
gh := gohttp.GoHttp[AccountResponse]{Request: request}
result, err := gh.HttpGet(context.Background())
if err != nil {
return nil, err
}
return result, nil
}
// all data -------------------------------
func (m *GoMeta) GetMarketAllDataByAccountId(req *RequestData, accountId string) (*AllDataResponse, error) {
api := m.Config.currentVersionBaseApi + ApiAccountAdsets
api = fmt.Sprintf(api, accountId)
req = m.handleRequest(req)
params, _ := query.Values(req)
request := &gohttp.Request{
Url: api,
ParamsValues: params,
}
gh := gohttp.GoHttp[AllDataResponse]{Request: request}
result, err := gh.HttpGet(context.Background())
if err != nil {
return nil, err
}
return result, nil
}
// 根据数据类型获取某个详情,如:广告组详情 广告详情 -------------------------------
func (m *GoMeta) GetMarketDetailByDataId(req *RequestData, dataId string) (*DataDetailResponse, error) {
api := m.Config.currentVersionBaseApi + ApiDataDetails
api = fmt.Sprintf(api, dataId)
req = m.handleRequest(req)
params, _ := query.Values(req)
request := &gohttp.Request{
Url: api,
ParamsValues: params,
}
gh := gohttp.GoHttp[DataDetailResponse]{
Request: request,
}
result, err := gh.HttpGet(context.Background())
if err != nil {
return nil, err
}
return result, nil
}
//---------------------------------------------------------------- 使用例子方法----------------------------------------------------------------
// 某个计划或者广告组所有详情数据以 国家小时为纬度的数据
// res.Paging.Cursors.After 通过这个参数重新请求下一页数据
func (m *GoMeta) GetMarketDetailsDataForCountry(outlineItem *AllDataItem, startDate, endDate string, pageSize int) (*DataDetailResponse, error) {
req := &RequestData{
Fields: adFields,
AccessToken: m.Config.AccessToken,
DateStart: startDate,
DateStop: endDate,
Limit: pageSize,
Breakdowns: "['country']", //默认以国家纬度数据请求
}
res, err := m.GetMarketDetailByDataId(req, outlineItem.Id)
if err != nil {
return nil, err
}
return res, nil
}
// 概要数据加载,下一页数据 res.Paging.Cursors.After
func (m *GoMeta) GetMarketAccountAdSetsOutline(accountId string, startDate, endDate string, pageSize int) (*AllDataResponse, error) {
req := &RequestData{
Fields: allDataFields,
AccessToken: m.Config.AccessToken,
DateStart: startDate,
DateStop: endDate,
Limit: pageSize,
}
res, err := m.GetMarketAllDataByAccountId(req, accountId)
if err != nil {
return nil, err
}
return res, nil
}
go-marketing/go-meta/market_types.go (6.7 KiB)
package gometa
import "github.com/gogf/gf/util/gconv"
// 广告系列,广告组,广告 字段通用(有冗余)
const (
accountFields = "id,account_id,name,account_status,balance,currency,business_name,business"
allDataFields = "name,objective,status,effective_status,campaign_id,account_id,adset_id" //拉取全部数据时用的字段
adFields = "adset_name,account_id,campaign_id,adset_id,ad_id,impressions,cpm,cpc,spend,clicks,objective,location,cost_per_unique_click,account_name,ctr,actions"
)
type CampaignStatus string
const (
ACTIVE CampaignStatus = "ACTIVE"
PAUSED CampaignStatus = "PAUSED"
DELETED CampaignStatus = "DELETED"
ARCHIVED CampaignStatus = "ARCHIVED"
)
// fb actions key
const (
OmniActivateApp = "omni_activate_app"
MobileAppInstall = "mobile_app_install"
OmniPurchase = "omni_purchase"
//messaging_first_reply = "onsite_conversion.messaging_first_reply"
//post_engagement = "post_engagement"
//page_engagement = "page_engagement"
//comment = "comment"
//messaging_conversation_started_7d = "onsite_conversion.messaging_conversation_started_7d"
//fb_mobile_activate_app = "app_custom_event.fb_mobile_activate_app"
//omni_app_install = "omni_app_install"
//video_view = "video_view"
//post_reaction = "post_reaction"
//link_click = "link_click"
//post_save = "onsite_conversion.post_save"
)
const (
ApiAccount = "/%s/client_ad_accounts"
ApiAccountCampaigns = "/act_%s/campaigns"
ApiAccountAdsets = "/act_%s/adsets"
ApiAccountAds = "/act_%s/ads"
ApiDataDetails = "/%s/insights"
ApiRefreshToken = "/oauth/access_token"
)
// 简要数据项(冗余:如:获取广告系列数据时 CampaignId,AdsetId 都为空 )
type AllDataItem struct {
Name string `json:"name,optional"`
Status string `json:"status,optional"`
EffectiveStatus string `json:"effective_status,optional"`
CampaignId string `json:"campaign_id,optional"`
AccountId string `json:"account_id,optional"`
AdsetId string `json:"adset_id,optional"`
Id string `json:"id,optional"`
}
type CampaignDetails struct {
AccountId string `json:"account_id,optional"`
CampaignId string `json:"campaign_id,optional"`
AdsetId string `json:"adset_id,optional"`
AdsetName string `json:"adset_name,optional"`
AdId string `json:"ad_id,optional"`
Impressions string `json:"impressions,optional"`
Cpm string `json:"cpm,optional"`
Cpc string `json:"cpc,optional"`
Spend string `json:"spend,optional"`
Clicks string `json:"clicks,optional"`
Conversions string `json:"conversions,optional"`
Objective string `json:"objective,optional"`
CostPerUniqueClick string `json:"cost_per_unique_click,optional"`
AccountName string `json:"account_name,optional"`
Ctr string `json:"ctr,optional"`
DateStart string `json:"date_start,optional"`
DateStop string `json:"date_stop,optional"`
Country string `json:"country,optional"`
Actions []struct {
ActionType string `json:"action_type,optional"`
Value string `json:"value,optional"`
} `json:"actions,optional"`
}
func (f *CampaignDetails) GetActionStat(actionType string) int64 {
for _, action := range f.Actions {
if action.ActionType == actionType {
return gconv.Int64(action.Value)
}
}
return 0
}
// facebook 投放账户信息结构
type Account struct {
Id string `json:"id,optional"`
AccountId string `json:"account_id,optional"`
Name string `json:"name,optional"`
AccountStatus int `json:"account_status,optional"`
Balance string `json:"balance,optional"`
Currency string `json:"currency,optional"`
//BusinessName string `json:"business_name,optional"`
//Business struct {
// Id string `json:"id,optional"`
// Name string `json:"name,optional"`
//} `json:"business,optional"`
}
type TokenItem struct {
AccessToken string `json:"access_token,optional"`
TokenType string `json:"token_type,optional"`
ExpiresIn int `json:"expires_in,optional"`
Error struct {
Message string `json:"message,optional"`
Type string `json:"type,optional"`
Code int `json:"code,optional"`
FbtraceId string `json:"fbtrace_id,optional"`
} `json:"error,optional"`
}
type ApiRefreshTokenRequest struct {
GrantType string `url:"grant_type"`
ClientId string `url:"client_id"`
ClientSecret string `url:"client_secret"`
FbExchangeToken string `url:"fb_exchange_token"`
}
type ApiAccessTokenRequest struct {
RedirectUri string `url:"redirect_uri"`
ClientId string `url:"client_id"`
ClientSecret string `url:"client_secret"`
Code string `url:"code,optional"`
}
type RequestData struct {
Limit int `url:"limit"`
//Before string `json:"before"`
After string `url:"after"`
AccessToken string `url:"access_token"`
Fields string `url:"fields"`
DateStart string `url:"date_start"`
DateStop string `url:"date_stop"`
TimeRange string `url:"time_range"` //DateStart And DateStop are here for query
TimeIncrement int `url:"time_increment"` // 1表示以天为单位
Timezone string `url:"time_zone"` //日期时区Asia/Shanghai
Breakdowns string `url:"breakdowns"` //['country'] 以国家纬度group by 查询
}
type RequestAccessKes struct {
AccessToken string `url:"access_token"`
}
type DetailsDataRequest struct {
RequestData
TimeIncrement int `url:"time_increment"`
Timezone string `url:"time_zone"` //Asia/Shanghai
Breakdowns string `url:"breakdowns"` //['country'] 以国家纬度group by 查询
}
type AccountResponse struct {
BaseResponse[[]Account]
}
type AllDataResponse struct {
BaseResponse[[]AllDataItem]
}
type DataDetailResponse struct {
BaseResponse[[]CampaignDetails]
}
type TokenResponse struct {
TokenItem
}
// ---------meta变现端--------------------------------
type EncryptedEcpmReq struct {
RequestId string `json:"request_id"`
Ecpms []string `json:"ecpms"`
AccessToken string `json:"access_token"`
SyncApi bool `json:"sync_api"`
}
type EncryptedEcpmRes struct {
RequestId string `json:"request_id"`
Success struct {
Value float64 `json:"value"`
Accuracy string `json:"accuracy"`
} `json:"success"`
Error struct {
Reason string `json:"reason"`
Description string `json:"description"`
NoImpressionCount int `json:"no_impression_count"`
InvalidImpressionCount int `json:"invalid_impression_count"`
} `json:"error"`
}
go-marketing/go-meta/test/gometa_test.go (1.0 KiB)
package main
import (
"context"
gocontext "github.com/gif-gif/go.io/go-context"
golog "github.com/gif-gif/go.io/go-log"
gometa "github.com/gif-gif/go.io/go-marketing/go-meta"
"github.com/gif-gif/go.io/goio"
"testing"
)
func TestAdmobApps(t *testing.T) {
goio.Init(goio.DEVELOPMENT)
err := gometa.Init(context.Background(), gometa.Config{
ClientId: "",
ClientSecret: "",
AccessToken: "",
})
if err != nil {
golog.Error(err)
return
}
// 测试获取应用列表
req := gometa.AudienceDataRequest{
AggregationPeriod: gometa.AGGREGATION_PERIOD_HOUR,
Since: "",
Until: "",
Breakdowns: []string{gometa.BREAKDOWN_COUNTRY, gometa.BREAKDOWN_PLACEMENT},
Metrics: gometa.DefaultAudienceMetrics,
Filter: []gometa.AudienceFilter{
{
Field: gometa.BREAKDOWN_PLACEMENT,
Operator: "in",
Values: []string{
"",
},
},
},
Limit: 10,
}
res, err := gometa.Default().GetAudienceReport(&req, "")
golog.Info(res)
<-gocontext.WithCancel().Done()
}
go-marketing/go-meta/test/test.json (3.4 KiB)
{
"data": [
{
"query_id": "531234567890123456789012345683d6",
"results": [
{
"time": "2021-08-06T07:00:00+0000",
"metric": "fb_ad_network_imp",
"breakdowns": [
{
"key": "placement",
"value": "123456789012345"
},
{
"key": "country",
"value": "AE"
}
],
"value": "1200"
},
{
"time": "2021-08-06T07:00:00+0000",
"metric": "fb_ad_network_imp",
"breakdowns": [
{
"key": "placement",
"value": "123456789012345"
},
{
"key": "country",
"value": "AU"
}
],
"value": "35"
},
{
"time": "2021-08-06T07:00:00+0000",
"metric": "fb_ad_network_revenue",
"breakdowns": [
{
"key": "placement",
"value": "123456789012345"
},
{
"key": "country",
"value": "AE"
}
],
"value": "21.212345"
},
{
"time": "2021-08-06T07:00:00+0000",
"metric": "fb_ad_network_request",
"breakdowns": [
{
"key": "placement",
"value": "123456789012345"
},
{
"key": "country",
"value": "AD"
}
],
"value": "1"
},
{
"time": "2021-08-06T07:00:00+0000",
"metric": "fb_ad_network_request",
"breakdowns": [
{
"key": "placement",
"value": "123456789012345"
},
{
"key": "country",
"value": "AE"
}
],
"value": "12"
},
{
"time": "2021-08-06T07:00:00+0000",
"metric": "fb_ad_network_click",
"breakdowns": [
{
"key": "placement",
"value": "123456789012345"
},
{
"key": "country",
"value": "AE"
}
],
"value": "1"
},
{
"time": "2021-08-06T07:00:00+0000",
"metric": "fb_ad_network_click",
"breakdowns": [
{
"key": "placement",
"value": "123456789012345"
},
{
"key": "country",
"value": "CA"
}
],
"value": "2"
}
],
"omitted_results": [
{
"time": "2021-08-06T07:00:00+0000",
"metric": "fb_ad_network_revenue",
"breakdowns": [
{
"key": "placement",
"value": "123456789012345"
},
{
"key": "country",
"value": "AU"
}
]
}
]
}
],
"paging": {
"cursors": {
"before": "MAZDZD",
"after": "MQZDZD"
},
"next": "https://graph.facebook.com/v10.0/142440604406900/adnetworkanalytics?access_token=<ACCESS_TOKEN>&since=2021-08-06&until=2021-08-06&breakdowns=%5B%22placement%22%2C%22country%22%5D&limit=2&metrics=%5B%22fb_ad_network_request%22%2C%22fb_ad_network_imp%22%2C%22fb_ad_network_click%22%2C%22fb_ad_network_revenue%22%5D&after=MQZDZD"
}
}
go-marketing/go-toutiao/ReadMe.md (24 B)
Toutiao Marketing Api
go-marketing/goattribution/appsflyer.go (1.5 KiB)
package goattribution
import (
"net/url"
"github.com/zeromicro/go-zero/core/logx"
)
/*
{
"af_tranid": "XW2mV9SveRKJ0p-mNip_hA",
"af_c_id": "ss_ads4eachs_android_HappyFruit2048_br_1208",
"af_adset_id": "1806913189",
"pid": "mintegral_int",
"af_prt": "ads4eachs",
"af_adset": "icon_512x512",
"af_ad": "icon_512x512",
"af_siteid": "mtg1145003732",
"af_ad_id": "1806913189",
"c": "ads4eachs_android_HappyFruit2048_br_1208"
}
*/
// type AppsFlyer struct {
// AfTranID string `json:"af_tranid"`
// AfCID string `json:"af_c_id"`
// AfAdsetID string `json:"af_adset_id"`
// Pid string `json:"pid"`
// AfPRT string `json:"af_prt"`
// AfAdset string `json:"af_adset"`
// AfAd string `json:"af_ad"`
// AfSiteID string `json:"af_siteid"`
// AfAdID string `json:"af_ad_id"`
// C string `json:"c"`
// }
type AppsFlyerAttributeHandler struct {
}
func (h *AppsFlyerAttributeHandler) Channel() string {
return "appsflyer"
}
func (h *AppsFlyerAttributeHandler) Match(queryParams url.Values) bool {
return len(queryParams.Get("af_tranid")) > 0
}
func (h *AppsFlyerAttributeHandler) Handle(queryParams url.Values) (*AttributeInfo, error) {
info, err := CreateAttributeInfo(queryParams, queryParams.Get("af_c_id"), queryParams.Get("c"))
if err != nil {
logx.Errorf("AppsFlyerAttributeHandler handle %v queryParams:%+v", err, queryParams)
}
// info.Channel = userdef.CHANNEL_APPSFLYER
info.Channel = info.CampaignChannel
info.CampaignName += "_" + queryParams.Get("af_adset")
return info, nil
}
go-marketing/goattribution/attribute.go (2.7 KiB)
package goattribution
import (
"errors"
"fmt"
golog "github.com/gif-gif/go.io/go-log"
"net/url"
"strings"
)
const (
CHANNEL_UNKWON = "unknown"
CHANNEL_ORGANIC = "organic"
CHANNEL_META = "meta"
CHANNEL_GOOGLE = "google"
CHANNEL_YANDEX = "yandex"
CHANNEL_APPSFLYER = "appsflyer"
CHANNEL_MINTEGRAL = "mintegral"
CHANNEL_BIGO = "bigo"
)
type AttributeHandler interface {
Match(queryParams url.Values) bool
Handle(queryParams url.Values) (*AttributeInfo, error)
Channel() string //开发者定义的渠道标识
}
type AttributeInfo struct {
ClickId string
Channel string
CampaignId string
CampaignName string
AdCostMode string
CampaignChannel string
CampaignPartner string
UtmCampaignId string
UtmCampaignName string
UtmSource string
UtmMedium string
UtmContent string
}
func CreateAttributeInfo(queryParams url.Values, campaignId, campaignName string) (*AttributeInfo, error) {
result := &AttributeInfo{
UtmSource: queryParams.Get("utm_source"),
UtmMedium: queryParams.Get("utm_medium"),
UtmContent: queryParams.Get("utm_content"),
ClickId: queryParams.Get("click_id"),
CampaignId: campaignId,
CampaignName: campaignName,
}
items := strings.Split(campaignName, "_")
if len(items) > 0 {
result.CampaignPartner = items[0]
}
if len(items) > 2 {
result.AdCostMode = items[2]
}
if len(items) > 3 {
result.CampaignChannel = items[3]
}
var err error
if len(items) < 4 {
err = fmt.Errorf("campaignName error:%s", campaignName)
}
return result, err
}
type Config struct {
Name string
DecryptKeys map[string]string // 每个平台 解密key -> [平台标识]=[解密key]
}
var __clients = map[string]*AttributeManager{}
// 可以一次初始化多个Redis实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("goattribution client [" + name + "] already exists")
}
__clients[name] = New(conf)
}
return
}
func GetClient(names ...string) *AttributeManager {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *AttributeManager {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("goattribution").Error("no default goattribution client")
return nil
}
go-marketing/goattribution/bigo.go (535 B)
package goattribution
import (
"net/url"
)
type BigoAttributeHandler struct {
}
func (h *BigoAttributeHandler) Channel() string {
return CHANNEL_BIGO
}
func (h *BigoAttributeHandler) Match(queryParams url.Values) bool {
utm_medium := queryParams.Get("utm_medium")
utm_source := queryParams.Get("utm_source")
return utm_source == h.Channel() || utm_medium == h.Channel()
}
func (h *BigoAttributeHandler) Handle(queryParams url.Values) (*AttributeInfo, error) {
return CreateBaseAttributeInfo(queryParams, h.Channel()), nil
}
go-marketing/goattribution/common.go (657 B)
package goattribution
import (
"net/url"
)
type CommonAttributeHandler struct {
_Channel string
}
func (h *CommonAttributeHandler) Channel() string {
return h._Channel
}
func (h *CommonAttributeHandler) Match(queryParams url.Values) bool {
return true
}
func (h *CommonAttributeHandler) Handle(queryParams url.Values) (*AttributeInfo, error) {
utm_medium := queryParams.Get("utm_medium")
utm_source := queryParams.Get("utm_source")
if utm_source != "" {
h._Channel = utm_source
} else if utm_medium != "" {
h._Channel = utm_medium
} else {
h._Channel = CHANNEL_UNKWON
}
return CreateBaseAttributeInfo(queryParams, h.Channel()), nil
}
go-marketing/goattribution/cryptography/aes_crt.go (431 B)
package cryptography
import (
"crypto/aes"
"crypto/cipher"
)
func AesCtrCrypt(plainText []byte, key, iv []byte) ([]byte, error) {
//1. 创建cipher.Block接口
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
//2. 创建分组模式,在crypto/cipher包中
stream := cipher.NewCTR(block, iv)
//3. 加密
dst := make([]byte, len(plainText))
stream.XORKeyStream(dst, plainText)
return dst, nil
}
go-marketing/goattribution/cryptography/aes_crt_test.go (2.9 KiB)
package cryptography
import (
"encoding/base64"
"encoding/hex"
"testing"
"github.com/stretchr/testify/assert"
)
func TestAesCtrCrypt(t *testing.T) {
type args struct {
plainText []byte
key []byte
iv []byte
}
key, _ := hex.DecodeString("BD3A6DE43FF080A512A00AE7B036134092A0600D1FAF0BF641183F95D4EEC001")
iv, _ := hex.DecodeString("8DA8F20226552F19F907767B3529C001")
tests := []struct {
name string
args args
want []byte
wantErr bool
}{
{
name: "1",
args: args{
// plainText: []byte(`{"app_code":"chatgpt","user_id":"","google_id":"google_id_test","android_id":"android_id_test","timezone_offset":28800,"hour":11,"country_code":"CN","adb":0,"mac":"mac","network":"","model":"","brand":"","os_rom":"","de_width":0,"de_height":0,"density":0,"wifi_proxy":0,"de_version":"","vc":0,"vn":"","request_time":0,"is_vpn":0,"sim":0,"boot_time":0,"language":"","wifi_mac":"","wifi_name":"","oaid":""}`),
plainText: []byte(`RxruQYor8EjHfNXQJIXgzAh6mv94fPcgKPLm82_UcJ-alU29yjOjML1WqsdH27JvsmAS6vweWzEv5CzCDvKaxAYrzAqk8tdeREyNzVJvLdd_9YkhE6wSXaaOU12rJMYCf4qCRmRJ9ibTDQZnJYTpOAHUhbbyfvU9OU4t-lyw9oIA3F3E0DChnxJr-9XHdhPhEaicn59jFydPyeAfxHuY2j8RVyjMREJVaGTyGgUTlu3ipUv9tMd_Kqq77p5VY6A`),
key: key,
iv: iv,
},
want: []byte{},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dbuf := make([]byte, base64.StdEncoding.DecodedLen(len(tt.args.plainText)))
n, err := base64.StdEncoding.Decode(dbuf, tt.args.plainText)
decodeString := dbuf[:n]
ciphertext, err := AesCtrCrypt(decodeString, tt.args.key, tt.args.iv)
buf := make([]byte, base64.StdEncoding.EncodedLen(len(ciphertext)))
base64.StdEncoding.Encode(buf, ciphertext)
val := string(buf)
_ = val
if (err != nil) != tt.wantErr {
t.Errorf("AesCtrCrypt() error = %v, wantErr %v", err, tt.wantErr)
return
}
})
}
}
func TestAesCtrCrypt2(t *testing.T) {
data := []byte(`{"user_id":"","referer":"utm_source=google-play&utm_medium=organic","ggClickTime":0,"request_time":1701087073,"country_code":"cn","vc":121,"vn":"1.2.1","google_id":"0e4735a1-4faf-4109-8834-5ccf5885cd7b","android_id":"43e7a62a4d32e320","hour":20,"timezone_offset":28800,"adb":true,"mac":"02:00:00:00:00:00","network":"wifi","model":"SM-G981U","brand":"samsung","os_rom":"samsung12","os_type":"android","de_width":360.0,"de_height":800.0,"density":4.0,"wifi_proxy":false,"de_version":"REL","is_vpn":false,"sim":true,"boot_time":2710244945,"language":"en"}`)
encryptKey, _ := hex.DecodeString(`BD3A6DE43FF080A512A00AE7B036134092A0600D1FAF0BF641183F95D4EEC800`)
encryptIv, _ := hex.DecodeString(`8DA8F20226552F19F907767B3529C800`)
dataEncrypt, err := AesCtrCrypt(data, encryptKey, encryptIv)
assert.NoError(t, err)
t.Log(dataEncrypt)
buf := make([]byte, base64.StdEncoding.EncodedLen(len(dataEncrypt)))
base64.StdEncoding.Encode(buf, dataEncrypt)
t.Log(buf)
}
go-marketing/goattribution/cryptography/aes_gcm.go (513 B)
package cryptography
import (
"crypto/aes"
"crypto/cipher"
"errors"
)
func AesGcmDecrypt(cipherText, key, nonce []byte) ([]byte, error) {
c, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(c)
if err != nil {
return nil, err
}
nonceSize := gcm.NonceSize()
if len(nonce) < nonceSize {
return nil, errors.New("cipherText too short")
}
// nonce, cipherText := cipherText[:nonceSize], cipherText[nonceSize:]
return gcm.Open(nil, nonce, cipherText, nil)
}
go-marketing/goattribution/cryptography/aes_gcm_test.go (3.1 KiB)
package cryptography
import (
"encoding/hex"
"testing"
"github.com/stretchr/testify/assert"
)
func TestAesGcmDecrypt(t *testing.T) {
//{"app":6316061275127559,"t":1687837737,"source":{"data":"fb718b9b023756cd09b388156eb82c345945e7ab590b817150d5049b3e9c6733d87dbe18a3cf67b0f239864cd30d3b762a33ec0efeef2f6a7a30e485261e4aab7f02ee995c19861ca4ab1e5c07dd2d63b9975059ed69e83e5b4b1d1dfae0bfed7cbe1b19a42513e48b5e1ff8e21f7d2c7a5193ded8bdb4588559437cce477da3101682deac44c376d5a6116b950b7ca9977b501abc3dc2860b5940b00bef074135fce6724eff96b64b8fcd13356142456f3a26f6ed0979c8b983872d33d33720617f7027212a8f0ac03155525236625c12571896251b533eb56b2f9e8504a947d2ebf8bb7b4e1084bbae1dbc4c1b9bf8ba60c776b3b393f084e9f443032e822aac1cfc2d9afe0137572c0be663667c072bb981a75a216ef0e3c5d421127d522b494569c19dbf220474e2fd067fd0d43a4fde5f366f00051fc9ffe198245b11a53fd4450bd7d15d5059cd48ee95d016077049a09c9a07370561a011bbf40a9f906515d1c9f73a322b82483a1e451f90cde8576664f535b1cc2fd49a903684315e5497084da96217b9c609fa4b36d7b0bebbfd8115933823","nonce":"47e6abd8736193d3e1f435bd"}}
key, err := hex.DecodeString("40a779689956f1b58771ad44a1d2bf160feab6269f933c90931b2ea9724f2a35")
assert.NoError(t, err)
// nonce, err := hex.DecodeString("47e6abd8736193d3e1f435bd")
nonce, err := hex.DecodeString("f49d0e32dedd8102a8e4c8cf")
assert.NoError(t, err)
// txt, err := hex.DecodeString("fb718b9b023756cd09b388156eb82c345945e7ab590b817150d5049b3e9c6733d87dbe18a3cf67b0f239864cd30d3b762a33ec0efeef2f6a7a30e485261e4aab7f02ee995c19861ca4ab1e5c07dd2d63b9975059ed69e83e5b4b1d1dfae0bfed7cbe1b19a42513e48b5e1ff8e21f7d2c7a5193ded8bdb4588559437cce477da3101682deac44c376d5a6116b950b7ca9977b501abc3dc2860b5940b00bef074135fce6724eff96b64b8fcd13356142456f3a26f6ed0979c8b983872d33d33720617f7027212a8f0ac03155525236625c12571896251b533eb56b2f9e8504a947d2ebf8bb7b4e1084bbae1dbc4c1b9bf8ba60c776b3b393f084e9f443032e822aac1cfc2d9afe0137572c0be663667c072bb981a75a216ef0e3c5d421127d522b494569c19dbf220474e2fd067fd0d43a4fde5f366f00051fc9ffe198245b11a53fd4450bd7d15d5059cd48ee95d016077049a09c9a07370561a011bbf40a9f906515d1c9f73a322b82483a1e451f90cde8576664f535b1cc2fd49a903684315e5497084da96217b9c609fa4b36d7b0bebbfd8115933823")
txt, err := hex.DecodeString("edb0f7406d835cec9163c32821b26cc31541d4a2c9d4e79f1924b5e9885e9625dbfa16d2ed79e004605d03991ac40bbc278b3493a1ffaf9a5577a92d6493dd7190d663dec053206053443647aa0d33d85bc1cd070b2983246c7418e7397647a14cf2e5f3a12a202d08384b0f10d4d0140c2337d9d3873d80ac0bda9d697b4122f67e26a3d0f9ecb38ed43506bd3cd3a8115dc0fc2c578949a66d223462896ea64e37b1c300bccd7bfbb235aaf3a9885293b939f2b17be7366d87041c9c0b4211788dc94a12267fe2ca22995c98c45dfb76972c0fe0ff229533d42dfa19dfdeba84f277bd04088bfe4d286ef48da09b9877b454323569a42d41183c2b85e055be226bc64c138ec3498ed2da5a73d1666457d9949780d3151c971c98ce98653fed074895d722bf0ee0557ab193885db33738d8a9192a96164852a22f0d52cf1953f3be4e7bbe08248925ce1cea08cfcc65c39a4f9f971f09b91596153f9b32d1f513d9ae63d800b686eca1f0dc46e360680aa7f1dd9adfe69b275b9dc06067f77dc3a7e846c2a51ba3e292a0fdd0934f5670eae2a9")
assert.NoError(t, err)
result, err := AesGcmDecrypt([]byte(txt), []byte(key), []byte(nonce))
assert.NoError(t, err)
t.Log(result)
}
go-marketing/goattribution/cryptography/hmacsha.go (358 B)
package cryptography
import (
"crypto/hmac"
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
)
func HMACSHA256(buf, key []byte) string {
h := hmac.New(sha256.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func HMACSHA512(buf, key []byte) string {
h := hmac.New(sha512.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
go-marketing/goattribution/cryptography/sorted_params.go (1001 B)
package cryptography
import (
"encoding/json"
"sort"
"strings"
)
func GetSortedString(info any, ignoreKeys map[string]struct{}, ignoreEmpty bool) (string, error) {
dataMap, ok := info.(map[string]string)
if !ok {
var newDataMap map[string]string
dataJson, err := json.Marshal(info)
if err != nil {
return "", err
}
err = json.Unmarshal(dataJson, &newDataMap)
if err != nil {
return "", err
}
dataMap = newDataMap
}
keys := make([]string, 0, len(dataMap)-1)
for key, val := range dataMap {
if _, ok := ignoreKeys[key]; ok {
continue
}
if ignoreEmpty {
if val == "" || val == "0" || val == "0.0" {
continue
}
}
keys = append(keys, key)
}
sort.Strings(keys)
builder := strings.Builder{}
for _, key := range keys {
builder.WriteString(key)
builder.WriteString("=")
builder.WriteString(dataMap[key])
builder.WriteString("&")
}
queryString := builder.String()
queryString = queryString[:len(queryString)-1]
return queryString, nil
}
go-marketing/goattribution/google.go (579 B)
package goattribution
import (
"net/url"
)
type GoogleAttributeHandler struct {
}
func (h *GoogleAttributeHandler) Channel() string {
return CHANNEL_GOOGLE
}
// gclid=123456789&utm_medium=referral&utm_source=apps.facebook.com&utm_campaign=fb4a&utm_content=bytedanceglobal_E.C.P.C&facebook_app_id=
func (h *GoogleAttributeHandler) Match(queryParams url.Values) bool {
return len(queryParams.Get("gclid")) > 0 //ads
}
func (h *GoogleAttributeHandler) Handle(queryParams url.Values) (*AttributeInfo, error) {
return CreateBaseAttributeInfo(queryParams, h.Channel()), nil
}
go-marketing/goattribution/manager.go (1.5 KiB)
package goattribution
import (
"fmt"
"slices"
)
type (
AttributeManager struct {
AttributeHandlers []AttributeHandler
Config Config
}
)
func New(config Config) *AttributeManager {
if config.DecryptKeys == nil {
config.DecryptKeys = make(map[string]string)
}
return &AttributeManager{
AttributeHandlers: []AttributeHandler{
&YandexAttributeHandler{},
&GoogleAttributeHandler{},
&FacebookAttributeHandler{
DecryptKey: config.DecryptKeys[CHANNEL_META],
},
&AppsFlyerAttributeHandler{},
&BigoAttributeHandler{},
&OrganicHandler{},
&CommonAttributeHandler{},
},
Config: config,
}
}
// 注册一个属性处理器
// 注意:注册的顺序很重要,因为会按照注册的顺序进行匹配
// 例如:如果先注册了 FacebookAttributeHandler,那么就会先匹配 FacebookAttributeHandler,然后再匹配 AppsFlyerAttributeHandler
// 这个函数会把handler插入到AttributeHandlers的头部
func (m *AttributeManager) AddAttributeHandler(handler AttributeHandler) {
m.AttributeHandlers = slices.Insert(m.AttributeHandlers, 0, handler)
}
func (m *AttributeManager) DecryptAttribute(referer string) (*AttributeInfo, error) {
queryParams, err := ParseQuery(referer)
if err != nil {
return nil, fmt.Errorf("parse query error: %w", err)
}
for _, handler := range m.AttributeHandlers {
if handler.Match(queryParams) {
v, err := handler.Handle(queryParams)
if err != nil {
continue
}
return v, nil
}
}
return &AttributeInfo{
Channel: "organic_unknown",
}, nil
}
go-marketing/goattribution/manager_test.go (5.9 KiB)
package goattribution
import (
"net/url"
"reflect"
"testing"
)
type TestAttributeHandler struct {
}
func (h *TestAttributeHandler) Channel() string {
return "CHANNEL_TEST"
}
func (h *TestAttributeHandler) Match(queryParams url.Values) bool {
val := queryParams.Get("utm_medium")
return val == h.Channel()
}
func (h *TestAttributeHandler) Handle(queryParams url.Values) (*AttributeInfo, error) {
return &AttributeInfo{
Channel: h.Channel(),
}, nil
}
func TestAttributeManager_DecryptAttribute(t *testing.T) {
type args struct {
appCode string
referer string
}
tests := []struct {
name string
args args
want *AttributeInfo
wantErr bool
}{
//{
// name: "userdef.CHANNEL_ORGANIC",
// args: args{
// appCode: "",
// referer: "utm_source=google-play\\u0026utm_medium=organic",
// },
// want: &AttributeInfo{
// Channel: "organic",
// },
// wantErr: false,
//},
{
name: "userdef.CHANNEL_GOOGLE",
args: args{
appCode: "",
referer: "gclid=123456789&utm_medium=referral&utm_source=apps.facebook.com&utm_campaign=fb4a&utm_content=bytedanceglobal_E.C.P.C&facebook_app_id=",
},
want: &AttributeInfo{
Channel: "google",
},
wantErr: false,
},
//{
// name: "userdef.CHANNEL_FACEBOOK",
// args: args{
// appCode: "walkgain",
// referer: "utm_source=apps.facebook.com\u0026utm_campaign=fb4a\u0026utm_content=%7B%22app%22%3A1442207956662894%2C%22t%22%3A1695907010%2C%22source%22%3A%7B%22data%22%3A%2291ffeebf5c95d0b41f2708122d93d1ea74b3dab359836c3ac70d4762404e9e186518220e888713d5df36ee9b2e53efc1625a404717ad4ff6a8619dac18e4fd6e34da71c91cfb4cc27657ff3b16f503043dbf5b884d77273a0803817ba390f7b0f1828e4d72110637f55eb327fe3df418f0d6fc4ba93bab3f850f47a5142ba79076ba27a816a9e15974c1b2d48681ccfb4a7a8af27c976ff486706ac4c33f7425b03d6fba30a5d5888dd64bc04bcdc42d1769876a93da585d9cc1900437f07120617db0768c220c8d2d594b5894923e42a406f4319bff1c217bc26078558ef2b5f472643fb95c3c58856840aee4219081639f2839c2e6668aff41aebb96156259fb97b96c94f2b9e48b09a6e611e66aa8a63dd4d9c19b8806637837c842714f9ae74cc8078b11470fbd6b69645b20bd97bf421361bc485db48602241578a15b1b86dcc90d926837a881ffc6092592ca9f62454fa3cd8ff5a801dc59ac03647b1b3c7133147767a56f1f42431815c5d007d0c3ed79ca77c4a7c0586161a83811463153f6b3c0e9c4e907c32f2f14c0556d3a307d3cd75e55%22%2C%22nonce%22%3A%22708b3b1a2cb59f5aaa6e10d7%22%7D%7D",
// },
// want: &AttributeInfo{
// Channel: "meta",
// CampaignId: "23858919684670718",
// CampaignName: "walkgain_cpi_fb_ether_zu_002",
// AdCostMode: "",
// CampaignChannel: "fb",
// CampaignPartner: "ether",
// },
// wantErr: false,
//},
//{
// name: "userdef.CHANNEL_MINTEGRAL",
// args: args{
// appCode: "walkgain",
// referer: "af_tranid\\u003dCtGOjOFjSEE-OXAljqyXHQ\\u0026af_c_id\\u003dss_ads4eachs_android_HappyFruit2048_br_1207\\u0026af_adset_id\\u003d1806894662\\u0026pid\\u003dmintegral_int\\u0026af_prt\\u003dads4eachs\\u0026af_adset\\u003dicon_512x512\\u0026af_ad\\u003dicon_512x512\\u0026af_siteid\\u003dmtg1132839114\\u0026af_ad_id\\u003d1806894662\\u0026c\\u003dads4eachs_happyfruit2048_cpi_mintegral_br_1207",
// },
// want: &AttributeInfo{
// Channel: "mintegral",
// CampaignId: "ss_ads4eachs_android_HappyFruit2048_br_1207",
// CampaignName: "ads4eachs_happyfruit2048_cpi_mintegral_br_1207_icon_512x512",
// AdCostMode: "cpi",
// CampaignChannel: "mintegral",
// CampaignPartner: "ads4eachs",
// },
// wantErr: false,
//}, {
// name: "userdef.CHANNEL_organicnot set",
// args: args{
// appCode: "walkgain",
// referer: "utm_source=(not%20set)&utm_medium=(not%20set)",
// },
// want: &AttributeInfo{
// Channel: "organic",
// CampaignId: "",
// CampaignName: "",
// AdCostMode: "",
// CampaignChannel: "",
// CampaignPartner: "",
// },
// wantErr: false,
//}, {
// name: "userdef.CHANNEL_TEST",
// args: args{
// appCode: "walkgain",
// referer: "utm_source=(not%20set)&utm_medium=CHANNEL_TEST",
// },
// want: &AttributeInfo{
// Channel: "CHANNEL_TEST",
// CampaignId: "",
// CampaignName: "",
// AdCostMode: "",
// CampaignChannel: "",
// CampaignPartner: "",
// },
// wantErr: false,
//}, {
// name: "userdef.CHANNEL_JUMP",
// args: args{
// appCode: "walkgain",
// referer: "utm_source=jump&utm_medium=jump",
// },
// want: &AttributeInfo{
// UtmSource: "jump",
// UtmMedium: "jump",
// Channel: "jump",
// CampaignId: "",
// CampaignName: "",
// AdCostMode: "",
// CampaignChannel: "",
// CampaignPartner: "",
// },
// wantErr: false,
//},
}
Init(Config{
Name: "default",
DecryptKeys: map[string]string{
CHANNEL_META: "key",
},
})
m := GetClient()
m.AddAttributeHandler(&TestAttributeHandler{})
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := m.DecryptAttribute(tt.args.referer)
if (err != nil) != tt.wantErr {
t.Errorf("AttributeManager.DecryptAttribute() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got.Channel, tt.want.Channel) {
t.Errorf("AttributeManager.DecryptAttribute() = %v, want %v", got.Channel, tt.want.Channel)
}
})
}
}
func TestDecryptAttribute(t *testing.T) {
Init(Config{
Name: "default",
DecryptKeys: map[string]string{
CHANNEL_META: "key",
},
})
m := GetClient()
type args struct {
appCode string
referer string
}
tests := []struct {
name string
args args
want *AttributeInfo
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := m.DecryptAttribute(tt.args.referer)
if (err != nil) != tt.wantErr {
t.Errorf("DecryptAttribute() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("DecryptAttribute() = %v, want %v", got, tt.want)
}
})
}
}
go-marketing/goattribution/meta.go (3.4 KiB)
package goattribution // Generated by https://quicktype.io
import (
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"github.com/gif-gif/go.io/go-marketing/goattribution/cryptography"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gogf/gf/util/gconv"
"github.com/zeromicro/go-zero/core/logx"
"net/url"
"strings"
)
type FacebookUtmContent struct {
App int64 `json:"app"`
T int64 `json:"t"`
Source struct {
Data string `json:"data"`
Nonce string `json:"nonce"`
} `json:"source"`
}
type FacebookAd struct {
AdID int64 `json:"ad_id"`
AdObjectiveName string `json:"ad_objective_name"`
AdGroupID int64 `json:"adgroup_id"`
AdGroupName string `json:"adgroup_name"` //ad名称
CampaignID int64 `json:"campaign_id"`
CampaignName string `json:"campaign_name"` //adset名称
CampaignGroupID int64 `json:"campaign_group_id"`
CampaignGroupName string `json:"campaign_group_name"` //campaign名称
AccountID int64 `json:"account_id"`
IsInstagram bool `json:"is_instagram"`
PublisherPlatform string `json:"publisher_platform"`
PlatformPosition interface{} `json:"platform_position"`
}
type FacebookAttributeHandler struct {
DecryptKey string
}
func (h *FacebookAttributeHandler) Channel() string {
return CHANNEL_META
}
func (h *FacebookAttributeHandler) Match(queryParams url.Values) bool {
return strings.Contains(queryParams.Get("utm_source"), h.Channel()) || strings.Contains(queryParams.Get("utm_source"), "facebook")
}
func (h *FacebookAttributeHandler) Handle(queryParams url.Values) (*AttributeInfo, error) {
key := []byte(h.DecryptKey)
info, err := h.DecryptFacebookAttribute(queryParams, queryParams.Get("utm_content"), key)
if err != nil {
//logx.Errorf("FacebookAttributeHandler handle DecryptFacebookAttribute error:%v queryParams:%+v", err, queryParams)
return nil, err
}
return info, nil
}
func (h *FacebookAttributeHandler) DecryptFacebookAttribute(queryParams url.Values, content string, key []byte) (*AttributeInfo, error) {
facebookAd, err := h.DecryptFacebookAd(content, key)
if err != nil {
return nil, err
}
info, err := CreateAttributeInfo(queryParams, gconv.String(facebookAd.CampaignID), facebookAd.CampaignName)
info.Channel = h.Channel()
return info, err
}
func (h *FacebookAttributeHandler) DecryptFacebookAd(content string, key []byte) (*FacebookAd, error) {
if len(key) == 0 {
return nil, errors.New("key cannot be empty")
}
var utmContent FacebookUtmContent
err := json.Unmarshal(goutils.StringToBytes(content), &utmContent)
if err != nil {
return nil, err
}
data := goutils.StringToBytes(utmContent.Source.Data)
n, err := hex.Decode(data, data)
if err != nil {
return nil, fmt.Errorf("decode data error:%v,data:%s", err, utmContent.Source.Data)
}
cipherText := data[:n]
nonceData := goutils.StringToBytes(utmContent.Source.Nonce)
n, err = hex.Decode(nonceData, nonceData)
if err != nil {
return nil, fmt.Errorf("decode Nonce error:%v,nonce:%s", err, utmContent.Source.Nonce)
}
nonce := nonceData[:n]
data, err = cryptography.AesGcmDecrypt(cipherText, key, nonce)
if err != nil {
return nil, fmt.Errorf("AesGcmDecrypt error:%v txt:%s key:%s nonce:%s", err, cipherText, key, nonce)
}
logx.Infof("DecryptFacebookAd:%s", data)
var result FacebookAd
err = json.Unmarshal(data, &result)
if err != nil {
return nil, err
}
return &result, nil
}
go-marketing/goattribution/organic.go (669 B)
package goattribution
import (
"net/url"
"strings"
)
type OrganicHandler struct {
}
func (h *OrganicHandler) Channel() string {
return CHANNEL_ORGANIC
}
func (h *OrganicHandler) Match(queryParams url.Values) bool {
utm_medium := strings.TrimSpace(queryParams.Get("utm_medium"))
utm_source := strings.TrimSpace(queryParams.Get("utm_source"))
return utm_medium == h.Channel() || utm_source == h.Channel() ||
(utm_source == "(not set)" && utm_medium == "(not set)") ||
(utm_source == "" && utm_medium == "")
}
func (h *OrganicHandler) Handle(queryParams url.Values) (*AttributeInfo, error) {
return CreateBaseAttributeInfo(queryParams, h.Channel()), nil
}
go-marketing/goattribution/readme.md (81 B)
解析识别推广referer字符串
支持自定义推广渠道识别添加
go-marketing/goattribution/referer.go (603 B)
package goattribution
import (
"fmt"
"net/url"
"strconv"
)
// referer示例:
//1. gclid=EAIaIQobChMIwLX_jcuziwMV1olQBh1cJASjEAEYASAAEgKDQPD_BwE&gbraid=0AAAAA-VTNA1RZmc_uqUMRcothWf8n9V1w&gad_source=5
//2. utm_source=(not%20set)&utm_medium=(not%20set)
//3. 客户端写死默认值 utm_source=google-play&utm_medium=organic
func ParseQuery(referer string) (url.Values, error) {
// 对Unicode字符解码
decodedStr, err := strconv.Unquote(`"` + referer + `"`)
if err != nil {
return nil, fmt.Errorf("decoding string error: %v", err)
}
// 解析URL参数
return url.ParseQuery(decodedStr)
}
go-marketing/goattribution/utils.go (498 B)
package goattribution
import "net/url"
func CreateBaseAttributeInfo(queryParams url.Values, channel string) *AttributeInfo {
clickId := queryParams.Get("click_id")
if clickId == "" {
clickId = queryParams.Get("clickid")
}
if clickId == "" {
clickId = queryParams.Get("sid")
}
return &AttributeInfo{
UtmSource: queryParams.Get("utm_source"),
UtmMedium: queryParams.Get("utm_medium"),
UtmContent: queryParams.Get("utm_content"),
ClickId: clickId,
Channel: channel,
}
}
go-marketing/goattribution/yandex.go (594 B)
package goattribution
import (
"net/url"
"strings"
)
type YandexAttributeHandler struct {
}
func (h *YandexAttributeHandler) Channel() string {
return CHANNEL_YANDEX
}
func (h *YandexAttributeHandler) Match(queryParams url.Values) bool {
utm_medium := strings.TrimSpace(queryParams.Get("utm_medium"))
utm_source := strings.TrimSpace(queryParams.Get("utm_source"))
return utm_medium == h.Channel() || utm_source == h.Channel()
}
func (h *YandexAttributeHandler) Handle(queryParams url.Values) (*AttributeInfo, error) {
return CreateBaseAttributeInfo(queryParams, h.Channel()), nil
}
go-message/README.md (638 B)
发送通知
飞书
普通群消息
gomessage.FeiShu(hookUrl, "这是普通的群消息")
钉钉
gomessage.InitDingDing("token","secret")
普通群消息
err := gomessage.DingDing("这是普通的群消息")
@特定人的消息
@对象必须为绑定钉钉的手机号
err := gomessage.DingDing("Lucy, Harvey, 你们的程序挂了", "18578924567", "+13414567890")
@所有人的消息
err := gomessage.DingDing("这是@所有人的消息", "*")
Telegram电报
gomessage.InitTelegram("token",false)
//chatId 个人ID或群组ID text 消息内容
err := gomessage.TelegramTo(chatId, "text")
go-message/dingding_test.go (664 B)
package gomessage
import "testing"
func TestDingDing(t *testing.T) {
InitDingDing("bb96f055f83a0ad78b3112ca849f29d37de5f1bfecc5d1e6f205e4f63e6b0e93", "SEC6fd7250c7e489eeda966719217d8bc45136819cfd73bee38e776769c563cacfe")
type args struct {
hookUrl string
text string
}
tests := []struct {
name string
args args
wantErr bool
}{
// TODO: Add test cases.
{name: "test", args: args{
text: "test",
}, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := DingDing(tt.args.text); (err != nil) != tt.wantErr {
t.Errorf("DingDing() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
go-message/feishu_test.go (622 B)
package gomessage
import "testing"
func TestFeiShu(t *testing.T) {
type args struct {
hookUrl string
text string
}
tests := []struct {
name string
args args
wantErr bool
}{
// TODO: Add test cases.
{name: "test", args: args{
hookUrl: "https://open.feishu.cn/open-apis/bot/v2/hook/aa0f28f1-1663-421b-9fa9-af9b0bbe2ca4",
text: "test",
}, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := FeiShu(tt.args.hookUrl, tt.args.text); (err != nil) != tt.wantErr {
t.Errorf("FeiShu() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
go-message/go_message.go (1.6 KiB)
package gomessage
import (
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/go-message/goding"
"github.com/gif-gif/go.io/go-message/gofeishu"
"github.com/gif-gif/go.io/go-message/gotg"
gotgbot "github.com/gif-gif/go.io/go-tgbot"
)
var __DingDing *goding.Webhook
var __Telegram *gotg.TelegramHook
// 全局初始化一个DingDing 发送器对象
func InitDingDing(accessToken string, secret string) {
__DingDing = CreateDingDing(accessToken, secret)
}
// 新创建一个DingDing 发送器对象
func CreateDingDing(accessToken string, secret string) *goding.Webhook {
ding := &goding.Webhook{
AccessToken: accessToken,
Secret: secret,
}
return ding
}
func DingDing(text string, at ...string) error {
if __DingDing == nil {
return nil
}
return __DingDing.SendMessageText(text, at...)
}
func FeiShu(hookUrl string, text string) error {
return gofeishu.FeiShu(hookUrl, text)
}
// telegram
// Telegram 发送器对象, inChina true时走代理IP
func InitTelegram(accessToken string, inChina bool) {
api := "https://api.telegram.com"
if inChina {
api = "https://tgapi.goio.dev"
}
pref := &gotgbot.TelegramBot{
Product: "goio",
Token: accessToken,
StartReply: "",
ApiUrl: api,
}
gobot, err := gotgbot.Create(pref)
if err != nil {
golog.WithTag("gomessage").Error(err.Error())
return
}
go gobot.StartBot()
__Telegram = &gotg.TelegramHook{
AccessToken: accessToken,
GotgBot: gobot,
}
}
// chatId 个人ID或群组ID text 消息内容
func Telegram(chatId int64, text string) error {
return __Telegram.SendMessageText(chatId, text)
}
go-message/goding/ding.go (3.6 KiB)
package goding
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"regexp"
"time"
)
type Webhook struct {
AccessToken string
Secret string
}
type response struct {
Code int `json:"errcode"`
Msg string `json:"errmsg"`
}
// SendMessageText Function to send message
//
//goland:noinspection GoUnhandledErrorResult
func (t *Webhook) SendMessageText(text string, at ...string) error {
msg := map[string]interface{}{
"msgtype": "text",
"text": map[string]string{
"content": text,
},
}
if len(at) == 0 {
} else if len(at) == 1 {
if at[0] == "*" { // at all
msg["at"] = map[string]interface{}{
"isAtAll": true,
}
} else { // at specific user
re := regexp.MustCompile(`^\+*\d{10,15}$`)
if re.MatchString(at[0]) {
msg["at"] = map[string]interface{}{
"atMobiles": at,
"isAtAll": false,
}
} else {
return errors.New(`parameter error, "at" parameter must be in "*" or mobile phone number format`)
}
}
} else {
re := regexp.MustCompile(`^\+*\d{10,15}$`)
for _, v := range at {
if !re.MatchString(v) {
return errors.New(`parameter error, "at" parameter must be in "*" or mobile phone number format`)
}
}
msg["at"] = map[string]interface{}{
"atMobiles": at,
"isAtAll": false,
}
}
b, err := json.Marshal(msg)
if err != nil {
return err
}
resp, err := http.Post(t.getURL(), "application/json", bytes.NewBuffer(b))
if err != nil {
return err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
var r response
err = json.Unmarshal(body, &r)
if err != nil {
return err
}
if r.Code != 0 {
return errors.New(fmt.Sprintf("response error: %s", string(body)))
}
return err
}
//goland:noinspection GoUnhandledErrorResult
func (t *Webhook) sendMessageMarkdown(title, text string, at ...string) error {
msg := map[string]interface{}{
"msgtype": "markdown",
"markdown": map[string]string{
"title": title,
"text": text,
},
}
if len(at) == 0 {
} else if len(at) == 1 {
if at[0] == "*" { // at all
msg["at"] = map[string]interface{}{
"isAtAll": true,
}
} else { // at specific user
re := regexp.MustCompile(`^\+*\d{10,15}$`)
if re.MatchString(at[0]) {
msg["at"] = map[string]interface{}{
"atMobiles": at,
"isAtAll": false,
}
} else {
return errors.New(`parameter error, "at" parameter must be in "*" or mobile phone number format`)
}
}
} else {
re := regexp.MustCompile(`^\+*\d{10,15}$`)
for _, v := range at {
if !re.MatchString(v) {
return errors.New(`parameter error, "at" parameter must be in "*" or mobile phone number format`)
}
}
msg["at"] = map[string]interface{}{
"atMobiles": at,
"isAtAll": false,
}
}
b, err := json.Marshal(msg)
if err != nil {
return err
}
resp, err := http.Post(t.getURL(), "application/json", bytes.NewBuffer(b))
if err != nil {
return err
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp.Body)
return err
}
func (t *Webhook) hmacSha256(stringToSign string, secret string) string {
h := hmac.New(sha256.New, []byte(secret))
h.Write([]byte(stringToSign))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func (t *Webhook) getURL() string {
wh := "https://oapi.dingtalk.com/robot/send?access_token=" + t.AccessToken
timestamp := time.Now().UnixNano() / 1e6
stringToSign := fmt.Sprintf("%d\n%s", timestamp, t.Secret)
sign := t.hmacSha256(stringToSign, t.Secret)
url := fmt.Sprintf("%s×tamp=%d&sign=%s", wh, timestamp, sign)
return url
}
go-message/gofeishu/feishu.go (818 B)
package gofeishu
import (
"errors"
"github.com/gif-gif/go.io/go-http/gohttpx"
goutils "github.com/gif-gif/go.io/go-utils"
"runtime"
"sync"
)
var (
__fieShuCH chan struct{}
__feiShuOnce sync.Once
)
func FeiShu(hookUrl string, text string) error {
if hookUrl == "" {
return nil
}
__feiShuOnce.Do(func() {
__fieShuCH = make(chan struct{}, runtime.NumCPU()*2)
})
__fieShuCH <- struct{}{}
defer func() { <-__fieShuCH }()
content := goutils.NewParams().Set("text", text)
params := goutils.NewParams().
Set("msg_type", "text").
Set("content", content.Data())
buf, err := gohttpx.PostJson(hookUrl, params.JSON())
if err != nil {
return err
}
rst, _ := goutils.Byte(buf).Params()
if msg := rst.Get("StatusMessage").String(); msg != "success" {
return errors.New(msg)
}
return nil
}
go-message/gotg/telegram.go (325 B)
package gotg
import gotgbot "github.com/gif-gif/go.io/go-tgbot"
type TelegramHook struct {
AccessToken string
GotgBot *gotgbot.GoTgBot
}
// SendMessageText Function to send message
func (t *TelegramHook) SendMessageText(chatId int64, text string) error {
_, err := t.GotgBot.SendMsgText(chatId, text)
return err
}
go-message/telegram_test.go (573 B)
package gomessage
import "testing"
func TestTelegram(t *testing.T) {
InitTelegram("7107568224:AAFgdiEsDqtFvBBScIfWku9IB8jr9Dpl-dw", true)
type args struct {
text string
}
tests := []struct {
name string
args args
wantErr bool
}{
// TODO: Add test cases.
{name: "test", args: args{
text: "test",
}, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := Telegram(5562314141, tt.args.text); (err != nil) != tt.wantErr {
t.Errorf("Telegram() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
go-mq/go-asynq/client.go (4.5 KiB)
package goasynq
import (
"encoding/json"
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
"github.com/hibiken/asynq"
"github.com/samber/lo"
"time"
)
type ClientConfig struct {
Config goredisc.Config `yaml:"Config" json:"config,optional"`
Name string `yaml:"Name" json:"name,optional"`
}
type GoAsynqClient struct {
Client *asynq.Client
Inspector *asynq.Inspector
//Redis *goredisc.GoRedisC
}
func NewClient(config ClientConfig) *GoAsynqClient {
if config.Config.PoolSize == 0 {
config.Config.PoolSize = 10
}
client := asynq.NewClient(asynq.RedisClientOpt{
Addr: config.Config.Addrs[0],
DB: config.Config.DB,
Password: config.Config.Password,
PoolSize: config.Config.PoolSize,
DialTimeout: lo.If(config.Config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.DialTimeout) * time.Second),
ReadTimeout: lo.If(config.Config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(config.Config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.WriteTimeout) * time.Second),
})
inspector := asynq.NewInspector(asynq.RedisClientOpt{
Addr: config.Config.Addrs[0],
Password: config.Config.Password,
DB: config.Config.DB,
PoolSize: config.Config.PoolSize,
DialTimeout: lo.If(config.Config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.DialTimeout) * time.Second),
ReadTimeout: lo.If(config.Config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(config.Config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.WriteTimeout) * time.Second),
})
//err := goredisc.Init(config.Config)
//if err != nil {
// logx.Errorf("init redis client error: %s", err)
//}
return &GoAsynqClient{
Client: client,
Inspector: inspector,
//Redis: goredisc.GetClient(config.Name),
}
}
func (c *GoAsynqClient) NewTask(taskTypeTopic string, payload any, opts ...asynq.Option) (*asynq.Task, error) {
payloadByte, err := json.Marshal(payload)
if err != nil {
return nil, err
}
return asynq.NewTask(taskTypeTopic, payloadByte, opts...), nil
}
func (c *GoAsynqClient) Enqueue(taskTypeTopic string, payload any, opts ...asynq.Option) (*asynq.TaskInfo, error) {
task, err := c.NewTask(taskTypeTopic, payload, opts...)
if err != nil {
return nil, err
}
info, err := c.Client.Enqueue(task)
if err != nil {
return nil, err
}
return info, nil
}
func (c *GoAsynqClient) Queues() ([]string, error) {
infos, err := c.Inspector.Queues()
if err != nil {
return nil, err
}
return infos, nil
}
func (c *GoAsynqClient) DeleteQueue(queueName string, force bool) error {
return c.Inspector.DeleteQueue(queueName, force)
}
func (c *GoAsynqClient) Close() error {
err := c.Inspector.Close()
if err != nil {
return err
}
err = c.Client.Close()
if err != nil {
return err
}
//
//if c.Redis != nil {
// return c.Redis.Close()
//}
return nil
}
//func (c *GoAsynqClient) DeleteQueueFromRedis(queueName string) error {
// // 创建 Redis 客户端
// rdb := c.Redis
// // Asynq 使用的 Redis key 模式
// patterns := []string{
// fmt.Sprintf("asynq:{%s}:*", queueName),
// fmt.Sprintf("asynq:queues:%s", queueName),
// // 其他相关的 key 模式
// }
//
// for _, pattern := range patterns {
// keys, err := rdb.Keys(pattern).Result()
// if err != nil {
// return err
// }
//
// if len(keys) > 0 {
// err = rdb.Del(keys...).Err()
// if err != nil {
// return err
// }
// }
// }
//
// return nil
//}
// 彻底地清理队列
func (c *GoAsynqClient) PurgeQueue(queueName string) error {
inspector := c.Inspector
// 获取队列信息
queues, err := inspector.Queues()
if err != nil {
return err
}
// 检查队列是否存在
queueExists := false
for _, q := range queues {
if q == queueName {
queueExists = true
break
}
}
if !queueExists {
return nil
}
// 删除各种状态的任务
deleteFuncs := []func(string) (int, error){
inspector.DeleteAllPendingTasks,
inspector.DeleteAllRetryTasks,
inspector.DeleteAllScheduledTasks,
inspector.DeleteAllArchivedTasks,
inspector.DeleteAllCompletedTasks,
}
for _, deleteFunc := range deleteFuncs {
if _, err := deleteFunc(queueName); err != nil {
// 继续删除其他状态的任务
return err
}
}
return c.DeleteQueue(queueName, true)
}
go-mq/go-asynq/go-asynqc/client.go (2.6 KiB)
package goasynqc
import (
"encoding/json"
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
"github.com/hibiken/asynq"
"github.com/samber/lo"
"time"
)
type GoAsynqClient struct {
Client *asynq.Client
}
type ClusterClientConfig struct {
Config goredisc.Config `yaml:"Config" json:"config,optional"`
Name string `yaml:"Name" json:"name,optional"`
}
func NewClusterClient(conf ClusterClientConfig) *GoAsynqClient {
config := conf.Config
var client *asynq.Client
if config.Type != "cluster" {
if conf.Config.PoolSize == 0 {
conf.Config.PoolSize = 10
}
client = asynq.NewClient(asynq.RedisClientOpt{
Addr: conf.Config.Addrs[0],
DB: conf.Config.DB,
Password: conf.Config.Password,
PoolSize: conf.Config.PoolSize,
DialTimeout: lo.If(conf.Config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.Config.DialTimeout) * time.Second),
ReadTimeout: lo.If(conf.Config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.Config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(conf.Config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(conf.Config.WriteTimeout) * time.Second),
})
} else {
client = asynq.NewClient(asynq.RedisClusterClientOpt{
Addrs: config.Addrs,
Password: config.Password,
MaxRedirects: lo.If(config.MaxRedirects <= 0, 10).Else(config.MaxRedirects),
DialTimeout: lo.If(config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.DialTimeout) * time.Second),
ReadTimeout: lo.If(config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.WriteTimeout) * time.Second),
})
}
return &GoAsynqClient{
Client: client,
}
}
func (c *GoAsynqClient) NewTask(taskTypeTopic string, payload any, opts ...asynq.Option) (*asynq.Task, error) {
payloadByte, err := json.Marshal(payload)
if err != nil {
return nil, err
}
return asynq.NewTask(taskTypeTopic, payloadByte, opts...), nil
}
func (c *GoAsynqClient) Enqueue(taskTypeTopic string, payload any, opts ...asynq.Option) (*asynq.TaskInfo, error) {
task, err := c.NewTask(taskTypeTopic, payload, opts...)
if err != nil {
return nil, err
}
info, err := c.Client.Enqueue(task)
if err != nil {
return nil, err
}
return info, nil
}
func (c *GoAsynqClient) Close() error {
err := c.Client.Close()
if err != nil {
return err
}
//
//if c.Redis != nil {
// return c.Redis.Close()
//}
return nil
}
go-mq/go-asynq/go-asynqc/goasynq_c.go (2.8 KiB)
package goasynqc
import (
"errors"
)
var __clients = map[string]*GoAsynqClient{}
var __servers = map[string]*GoAsynqServer{}
var __inspector = map[string]*GoAsynqInspector{}
// client for cluster or node
func InitClient(configs ...ClusterClientConfig) error {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name] = NewClusterClient(conf)
}
return nil
}
func GetClient(names ...string) *GoAsynqClient {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DefaultClient() *GoAsynqClient {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
// server for cluster or node
func InitServer(configs ...ClusterServerConfig) error {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__servers[name] = ClusterRunServer(conf)
}
return nil
}
func GetServer(names ...string) *GoAsynqServer {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __servers[name]; ok {
return cli
}
return nil
}
func DelServer(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__servers, name)
}
}
}
func DefaultServer() *GoAsynqServer {
if cli, ok := __servers["default"]; ok {
return cli
}
if l := len(__servers); l == 1 {
for _, cli := range __servers {
return cli
}
}
return nil
}
// __inspector for cluster or node
func InitInspector(configs ...ClusterInspectorConfig) error {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __inspector[name] != nil {
return errors.New("__inspector already exists")
}
__inspector[name] = NewClusterInspector(conf)
}
return nil
}
func GetInspector(names ...string) *GoAsynqInspector {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __inspector[name]; ok {
return cli
}
return nil
}
func DelInspector(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__inspector, name)
}
}
}
func DefaultInspector() *GoAsynqInspector {
if cli, ok := __inspector["default"]; ok {
return cli
}
if l := len(__inspector); l == 1 {
for _, cli := range __inspector {
return cli
}
}
return nil
}
go-mq/go-asynq/go-asynqc/inspector.go (3.5 KiB)
package goasynqc
import (
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
"github.com/hibiken/asynq"
"github.com/samber/lo"
"time"
)
type GoAsynqInspector struct {
Inspector *asynq.Inspector
}
type ClusterInspectorConfig struct {
Config goredisc.Config `yaml:"Config" json:"config,optional"`
Name string `yaml:"Name" json:"name,optional"`
}
func NewClusterInspector(conf ClusterInspectorConfig) *GoAsynqInspector {
config := conf.Config
var inspector *asynq.Inspector
if config.Type != "cluster" {
inspector = asynq.NewInspector(asynq.RedisClientOpt{
Addr: config.Addrs[0],
Password: config.Password,
DB: config.DB,
PoolSize: config.PoolSize,
DialTimeout: lo.If(config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.DialTimeout) * time.Second),
ReadTimeout: lo.If(config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.WriteTimeout) * time.Second),
})
} else {
inspector = asynq.NewInspector(asynq.RedisClusterClientOpt{
Addrs: config.Addrs,
Password: config.Password,
DialTimeout: lo.If(config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.DialTimeout) * time.Second),
ReadTimeout: lo.If(config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.WriteTimeout) * time.Second),
})
}
return &GoAsynqInspector{
Inspector: inspector,
}
}
func (c *GoAsynqInspector) Close() error {
err := c.Inspector.Close()
if err != nil {
return err
}
//
//if c.Redis != nil {
// return c.Redis.Close()
//}
return nil
}
//func (c *GoAsynqInspector) DeleteQueueFromRedis(queueName string) error {
// // 创建 Redis 客户端
// rdb := c.Redis
// // Asynq 使用的 Redis key 模式
// patterns := []string{
// fmt.Sprintf("asynq:{%s}:*", queueName),
// fmt.Sprintf("asynq:queues:%s", queueName),
// // 其他相关的 key 模式
// }
//
// for _, pattern := range patterns {
// keys, err := rdb.Keys(pattern).Result()
// if err != nil {
// return err
// }
//
// if len(keys) > 0 {
// err = rdb.Del(keys...).Err()
// if err != nil {
// return err
// }
// }
// }
//
// return nil
//}
// 彻底地清理队列
func (c *GoAsynqInspector) PurgeQueue(queueName string) error {
inspector := c.Inspector
// 获取队列信息
queues, err := inspector.Queues()
if err != nil {
return err
}
// 检查队列是否存在
queueExists := false
for _, q := range queues {
if q == queueName {
queueExists = true
break
}
}
if !queueExists {
return nil
}
// 删除各种状态的任务
deleteFuncs := []func(string) (int, error){
inspector.DeleteAllPendingTasks,
inspector.DeleteAllRetryTasks,
inspector.DeleteAllScheduledTasks,
inspector.DeleteAllArchivedTasks,
inspector.DeleteAllCompletedTasks,
}
for _, deleteFunc := range deleteFuncs {
if _, err := deleteFunc(queueName); err != nil {
// 继续删除其他状态的任务
return err
}
}
return c.DeleteQueue(queueName, true)
}
func (c *GoAsynqInspector) DeleteQueue(queueName string, force bool) error {
return c.Inspector.DeleteQueue(queueName, force)
}
func (c *GoAsynqInspector) Queues() ([]string, error) {
infos, err := c.Inspector.Queues()
if err != nil {
return nil, err
}
return infos, nil
}
go-mq/go-asynq/go-asynqc/server.go (4.4 KiB)
package goasynqc
import (
"context"
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/hibiken/asynq"
"github.com/samber/lo"
"time"
)
type GoAsynqServer struct {
ServeMux *asynq.ServeMux
Server *asynq.Server
}
type ClusterServerConfig struct {
Name string `yaml:"Name" json:"name,optional"`
Config goredisc.Config `yaml:"Config" json:"config,optional"`
Concurrency int `yaml:"Concurrency" json:"concurrency,optional"` //default 10 指定要使用的并发工作线程数量
Queues map[string]int `yaml:"Queues" json:"queues,optional"`
}
func ClusterRunServer(conf ClusterServerConfig) *GoAsynqServer {
config := conf.Config
server := &asynq.Server{}
if config.Type != "cluster" {
server = RunServerByNode(conf)
} else {
server = RunServer(conf)
}
// mux maps a type to a handler
mux := asynq.NewServeMux()
gs := &GoAsynqServer{
ServeMux: mux,
Server: server,
}
goutils.AsyncFunc(func() { // 异步运行挂起
defer gs.Stop()
if err := server.Run(mux); err != nil {
golog.WithTag("GoAsynqServer").Error("could not run server: %v", err)
}
golog.WithTag("GoAsynqServer").Info("stop running")
})
return gs
}
func RunServer(conf ClusterServerConfig) *asynq.Server {
if conf.Concurrency == 0 {
conf.Concurrency = 10
}
if conf.Queues == nil {
conf.Queues = map[string]int{
"critical": 6,
"default": 3,
"low": 1,
}
}
config := conf.Config
server := asynq.NewServer(
asynq.RedisClusterClientOpt{
Addrs: config.Addrs,
Password: config.Password,
DialTimeout: lo.If(config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.DialTimeout) * time.Second),
ReadTimeout: lo.If(config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.WriteTimeout) * time.Second),
},
asynq.Config{
// Specify how many concurrent workers to use
Concurrency: conf.Concurrency,
// Optionally specify multiple queues with different priority.
Queues: conf.Queues,
// See the godoc for other configuration options
},
)
return server
}
func RunServerByNode(config ClusterServerConfig) *asynq.Server {
if config.Config.PoolSize == 0 {
config.Config.PoolSize = 10
}
if config.Concurrency == 0 {
config.Concurrency = 10
}
if config.Queues == nil {
config.Queues = map[string]int{
"critical": 6,
"default": 3,
"low": 1,
}
}
srv := asynq.NewServer(
asynq.RedisClientOpt{
Addr: config.Config.Addrs[0],
Password: config.Config.Password,
DB: config.Config.DB,
PoolSize: config.Config.PoolSize,
DialTimeout: lo.If(config.Config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.DialTimeout) * time.Second),
ReadTimeout: lo.If(config.Config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(config.Config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.WriteTimeout) * time.Second),
},
asynq.Config{
// Specify how many concurrent workers to use
Concurrency: config.Concurrency,
// Optionally specify multiple queues with different priority.
Queues: config.Queues,
// See the godoc for other configuration options
},
)
return srv
}
func (s *GoAsynqServer) HandleFunc(taskTypeTopic string, handler func(context.Context, *asynq.Task) error) {
s.ServeMux.HandleFunc(taskTypeTopic, handler)
}
func (s *GoAsynqServer) Handle(taskTypeTopic string, handler asynq.Handler) {
s.ServeMux.Handle(taskTypeTopic, handler)
}
// Stop指示服务器停止从队列中提取新任务。
// 在关闭服务器之前,可以使用Stop来确保所有
// 在服务器关闭之前处理当前活动的任务。
//
// Stop不会关闭服务器,请确保在退出前调用shutdown。
func (s *GoAsynqServer) Stop() {
s.Server.Stop()
}
// Shutdown会优雅地关闭服务器。
// 它优雅地关闭了所有活跃的员工。服务器将等待
// 在配置中指定的持续时间内,主动工作人员完成处理任务。关机超时。
// 如果worker在超时期间没有完成任务处理,则该任务将被推回Redis。
func (s *GoAsynqServer) Shutdown() {
s.Server.Shutdown()
}
go-mq/go-asynq/goasynq.go (1.8 KiB)
package goasynq
import "errors"
var __clients = map[string]*GoAsynqClient{}
var __servers = map[string]*GoAsynqServer{}
// client
func InitClient(configs ...ClientConfig) error {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name] = NewClient(conf)
}
return nil
}
func GetClient(names ...string) *GoAsynqClient {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DefaultClient() *GoAsynqClient {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
// server
func InitServer(configs ...ServerConfig) error {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__servers[name] = RunServer(conf)
}
return nil
}
func GetServer(names ...string) *GoAsynqServer {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __servers[name]; ok {
return cli
}
return nil
}
func DelServer(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__servers, name)
}
}
}
func DefaultServer() *GoAsynqServer {
if cli, ok := __servers["default"]; ok {
return cli
}
if l := len(__servers); l == 1 {
for _, cli := range __servers {
return cli
}
}
return nil
}
go-mq/go-asynq/readme.md (271 B)
轻量级队列基于redis
-
goasynq 包 已废弃, 改用 goasynqc 包兼容单点和集群模式
-
延时队列 定时队列 消息队列(适合不是高并发场景)
- 基于 https://github.com/hibiken/asynq
- 监控界面 https://github.com/hibiken/asynqmon
go-mq/go-asynq/server.go (3.2 KiB)
package goasynq
import (
"context"
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/hibiken/asynq"
"github.com/samber/lo"
"time"
)
type ServerConfig struct {
Name string `yaml:"Name" json:"name,optional"`
Config goredisc.Config `yaml:"Config" json:"config,optional"`
Concurrency int `yaml:"Concurrency" json:"concurrency,optional"` //default 10 指定要使用的并发工作线程数量
Queues map[string]int `yaml:"Queues" json:"queues,optional"`
}
type GoAsynqServer struct {
ServeMux *asynq.ServeMux
Server *asynq.Server
}
// Stop指示服务器停止从队列中提取新任务。
// 在关闭服务器之前,可以使用Stop来确保所有
// 在服务器关闭之前处理当前活动的任务。
//
// Stop不会关闭服务器,请确保在退出前调用shutdown。
func (s *GoAsynqServer) Stop() {
s.Server.Stop()
}
// Shutdown会优雅地关闭服务器。
// 它优雅地关闭了所有活跃的员工。服务器将等待
// 在配置中指定的持续时间内,主动工作人员完成处理任务。关机超时。
// 如果worker在超时期间没有完成任务处理,则该任务将被推回Redis。
func (s *GoAsynqServer) Shutdown() {
s.Server.Shutdown()
}
func RunServer(config ServerConfig) *GoAsynqServer {
if config.Config.PoolSize == 0 {
config.Config.PoolSize = 10
}
if config.Concurrency == 0 {
config.Concurrency = 10
}
if config.Queues == nil {
config.Queues = map[string]int{
"critical": 6,
"default": 3,
"low": 1,
}
}
srv := asynq.NewServer(
asynq.RedisClientOpt{
Addr: config.Config.Addrs[0],
Password: config.Config.Password,
DB: config.Config.DB,
PoolSize: config.Config.PoolSize,
DialTimeout: lo.If(config.Config.DialTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.DialTimeout) * time.Second),
ReadTimeout: lo.If(config.Config.ReadTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.ReadTimeout) * time.Second),
WriteTimeout: lo.If(config.Config.WriteTimeout <= 0, time.Duration(5)*time.Second).Else(time.Duration(config.Config.WriteTimeout) * time.Second),
},
asynq.Config{
// Specify how many concurrent workers to use
Concurrency: config.Concurrency,
// Optionally specify multiple queues with different priority.
Queues: config.Queues,
// See the godoc for other configuration options
},
)
// mux maps a type to a handler
mux := asynq.NewServeMux()
gs := &GoAsynqServer{
ServeMux: mux,
Server: srv,
}
goutils.AsyncFunc(func() { // 异步运行挂起
defer gs.Stop()
if err := srv.Run(mux); err != nil {
golog.WithTag("GoAsynqServer").Error("could not run server: %v", err)
}
golog.WithTag("GoAsynqServer").Info("stop running")
})
return gs
}
func (s *GoAsynqServer) HandleFunc(taskTypeTopic string, handler func(context.Context, *asynq.Task) error) {
s.ServeMux.HandleFunc(taskTypeTopic, handler)
}
func (s *GoAsynqServer) Handle(taskTypeTopic string, handler asynq.Handler) {
s.ServeMux.Handle(taskTypeTopic, handler)
}
go-mq/go-kafka/adapter.go (1007 B)
package gokafka
import "github.com/IBM/sarama"
// 生产者
type IProducer interface {
Client() sarama.Client
// 发送消息 - 同步
SendMessage(msg IMessage) (partition int32, offset int64, err error)
SendMessages(msgs []IMessage) (err error)
// 发送消息 - 异步
SendAsyncMessage(msg IMessage, cb MessageHandler) (err error)
SendAsyncMessages(msgs []IMessage, cb MessageHandler) (err error)
}
// 消费者
type IConsumer interface {
Client() sarama.Client
// 从指定分区消费
WithPartition(partition int32) IConsumer
// 从指定位置开始
WithOffset(offset int64) IConsumer
// 从最新位置开始
WithOffsetNewest() IConsumer
// 从头开始
WithOffsetOldest() IConsumer
// 消费
Consume(topic string, handler ConsumerHandler)
// 分组topic
ConsumeGroup(groupId string, topics []string, handler ConsumerHandler)
}
// 消息
type IMessage interface {
Topic() string
Key() string
Headers() map[string]string
Serialize() []byte
Deserialize(b []byte)
}
go-mq/go-kafka/adapter_consumer.go (3.2 KiB)
package gokafka
import (
"context"
"errors"
"github.com/IBM/sarama"
gocontext "github.com/gif-gif/go.io/go-context"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"time"
)
type consumer struct {
*GoKafka
hasSetPartition bool // 是否设置分区
partition int32 // 分区
offset int64
}
func (c *consumer) Client() sarama.Client {
return c.GoKafka.Client
}
// 设置 分区
func (c *consumer) WithPartition(partition int32) IConsumer {
c.hasSetPartition = true
c.partition = partition
return c
}
// 设置 起始位置
func (c *consumer) WithOffset(offset int64) IConsumer {
c.offset = offset
return c
}
// 设置 起始位置 = 最新位置
func (c *consumer) WithOffsetNewest() IConsumer {
c.offset = sarama.OffsetNewest
return c
}
// 设置 起始位置 = 从头开始
func (c *consumer) WithOffsetOldest() IConsumer {
c.offset = sarama.OffsetOldest
return c
}
// 消费消息,默认处理最新消息
func (c *consumer) Consume(topic string, handler ConsumerHandler) {
log := golog.WithTag("gokafka-consumer").WithField("topic", topic)
consumer, err := sarama.NewConsumerFromClient(c.Client())
if err != nil {
log.Error(err)
return
}
defer func() {
if err := consumer.Close(); err != nil {
log.Error(err)
}
}()
if c.offset == 0 {
c.offset = sarama.OffsetNewest
}
pc, err := consumer.ConsumePartition(topic, c.partition, c.offset)
if err != nil {
log.Error(err)
return
}
defer func() {
if err := pc.Close(); err != nil {
log.Error(err)
}
}()
for {
select {
case <-gocontext.WithCancel().Done():
log.Debug("Context被取消,停止消费")
return
case err := <-pc.Errors():
if err != nil {
log.Error(err)
}
case msg, ok := <-pc.Messages():
if !ok {
log.Debug("消息通道被关闭,停止消费")
return
}
ctx := gocontext.WithLog()
ctx.Log.WithTag("gokafka-consumer").WithField("msg", msg)
if err = handler(ctx, &ConsumerMessage{ConsumerMessage: msg}, nil); err != nil {
log.Error(err)
}
// 删除缓存
key := string(msg.Key)
if c.redis != nil {
c.redis.Del(key)
}
}
}
}
// 分组
func (c *consumer) ConsumeGroup(groupId string, topics []string, handler ConsumerHandler) {
l := golog.WithTag("gokafka-consumer-group").
WithField("groupId", groupId).
WithField("topics", topics)
cg, err := sarama.NewConsumerGroupFromClient(groupId, c.GoKafka.Client)
if err != nil {
l.Error(err)
return
}
defer func() {
if c.GoKafka.Client != nil {
c.GoKafka.Client.Close()
l.Debug("client 退出")
}
}()
defer func() {
cg.Close()
l.Debug("consumer-group 退出")
}()
var (
done = make(chan struct{})
flag bool
)
ctx, cancel := context.WithCancel(context.Background())
goutils.AsyncFunc(func() {
for {
select {
case err := <-cg.Errors():
if err != nil {
l.Error(err)
}
default:
err := cg.Consume(ctx, topics, group{id: groupId, handler: handler, GoKafka: c.GoKafka})
if err != nil && !errors.Is(err, sarama.ErrClosedConsumerGroup) {
l.Error(err)
}
if flag {
done <- struct{}{}
return
}
}
}
})
select {
case <-gocontext.WithCancel().Done():
flag = true
cancel()
}
<-done
time.Sleep(time.Second)
}
go-mq/go-kafka/adapter_consumer_group.go (3.3 KiB)
package gokafka
import (
"encoding/json"
"fmt"
"github.com/IBM/sarama"
gocontext "github.com/gif-gif/go.io/go-context"
goutils "github.com/gif-gif/go.io/go-utils"
"time"
)
// 分组
type group struct {
*GoKafka
id string
handler ConsumerHandler
}
func (g group) Setup(sarama.ConsumerGroupSession) error {
return nil
}
func (group) Cleanup(sarama.ConsumerGroupSession) error {
return nil
}
func (g group) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
for {
select {
case <-session.Context().Done():
return fmt.Errorf("关闭会话上下文: %s", session.Context().Err())
case msg, ok := <-claim.Messages():
if !ok {
return fmt.Errorf("消费通道关闭: groupId=%s topic=%s partition=%d", g.id, claim.Topic(), claim.Partition())
}
g.doHandler(msg, session)
}
}
}
func (g group) doHandler(msg *sarama.ConsumerMessage, session sarama.ConsumerGroupSession) (err error) {
// 消息key
key := string(msg.Key)
// 消息试题
m := goutils.M{
"topic": msg.Topic,
"key": key,
"partition": msg.Partition,
"offset": msg.Offset,
"timestamp": msg.Timestamp.Format("2006-01-02 15:04:05"),
}
// 填充数据
{
// body
if len(msg.Value) > 0 {
var body interface{}
if err = json.Unmarshal(msg.Value, &body); err == nil {
m["body"] = body
} else {
m["body"] = string(msg.Value)
}
}
// headers
for _, i := range msg.Headers {
var headers = map[string]string{}
headers[string(i.Key)] = string(i.Value)
m["headers"] = headers
}
}
// 定义上下文
ctx := gocontext.WithLog()
ctx.Log.WithTag("gokafka-consumer-group", g.id).WithField("msg", m)
if g.redis == nil {
ctx.Log.WithField("msg", m)
g.handleMsg(ctx, msg, session)
return nil
}
// uniq key
{
var uniqKey string
if key != "" {
uniqKey = fmt.Sprintf("%s:%s", g.id, key)
} else {
uniqKey = fmt.Sprintf("%s:%s:%s", g.id, msg.Topic, goutils.MD5([]byte(g.id+msg.Topic+string(msg.Value))))
}
if g.redis != nil {
ok := g.redis.SetNX(uniqKey, goutils.M{
"topic": msg.Topic,
"body": m["body"],
"headers": m["headers"],
"timestamp": m["timestamp"],
}.String(), 300*time.Second).Val()
if !ok {
ctx.Log.Warn("消息消费失败,并发消费")
return
}
defer func() {
g.redis.Del(uniqKey)
}()
}
}
// 建立缓存
if g.redis != nil && key != "" {
g.redis.Set1(key, goutils.M{
"topic": msg.Topic,
"body": m["body"],
"headers": m["headers"],
"timestamp": m["timestamp"],
}.String(), time.Hour)
}
// 打印日志
t1 := time.Now()
defer func() {
ctx.Log.WithField("执行时间", fmt.Sprintf("%f", float64(time.Now().Sub(t1).Milliseconds())/1e3))
if err != nil {
ctx.Log.Error("消息消费失败", err)
return
}
//ctx.Log.Debug("消息消费成功")
}()
// 执行业务方法
g.handleMsg(ctx, msg, session)
// 删除缓存
if g.redis != nil && key != "" {
g.redis.Del(key)
}
return
}
func (g group) handleMsg(ctx *gocontext.Context, msg *sarama.ConsumerMessage, session sarama.ConsumerGroupSession) {
// 执行业务方法
if err := g.handler(ctx, &ConsumerMessage{ConsumerMessage: msg, GroupSession: session}, nil); err != nil {
return
}
// 提交
session.MarkMessage(msg, "")
}
go-mq/go-kafka/adapter_producer.go (5.7 KiB)
package gokafka
import (
"errors"
"github.com/IBM/sarama"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"time"
)
type producer struct {
*GoKafka
focus bool // 是否强制发送
}
func (p *producer) Client() sarama.Client {
return p.GoKafka.Client
}
// 发送消息 - 同步- 批量
func (p *producer) SendMessages(msgs []IMessage) (err error) {
if len(msgs) == 0 {
return nil
}
var sendMsgs []*sarama.ProducerMessage
for _, msg := range msgs {
m := &sarama.ProducerMessage{
Topic: msg.Topic(),
Value: sarama.ByteEncoder(msg.Serialize()),
}
if v := msg.Key(); v != "" {
m.Key = sarama.StringEncoder(v)
}
if data := msg.Headers(); data != nil {
var headers []sarama.RecordHeader
for k, v := range data {
headers = append(headers, sarama.RecordHeader{
Key: []byte(k),
Value: []byte(v),
})
}
m.Headers = headers
}
sendMsgs = append(sendMsgs, m)
}
defer func() {
log := golog.WithTag("gokafka-producer").WithField("msg", goutils.M{
"send_msgs_length": len(sendMsgs),
})
if err != nil {
log.Error("消息发送失败", err)
return
}
//log.Debug("消息发送成功")
}()
var producer sarama.SyncProducer
producer, err = sarama.NewSyncProducerFromClient(p.Client())
if err != nil {
return
}
defer producer.Close()
err = producer.SendMessages(sendMsgs)
return
}
// 发送消息 - 同步
func (p *producer) SendMessage(msg IMessage) (partition int32, offset int64, err error) {
m := &sarama.ProducerMessage{
Topic: msg.Topic(),
Value: sarama.ByteEncoder(msg.Serialize()),
}
if v := msg.Key(); v != "" {
m.Key = sarama.StringEncoder(v)
}
if data := msg.Headers(); data != nil {
var headers []sarama.RecordHeader
for k, v := range data {
headers = append(headers, sarama.RecordHeader{
Key: []byte(k),
Value: []byte(v),
})
}
m.Headers = headers
}
defer func() {
log := golog.WithTag("gokafka-producer").WithField("msg", goutils.M{
"topic": msg.Topic(),
"key": msg.Key(),
"headers": msg.Headers(),
"body": msg,
"partition": m.Partition,
"offset": m.Offset,
})
if err != nil {
log.Error("消息发送失败", err)
return
}
//log.Debug("消息发送成功")
}()
// 添加缓存
if p.redis != nil && len(msg.Key()) > 0 {
if p.focus {
p.redis.Del(msg.Key())
}
if p.redis.Exists(msg.Key()).Val() > 0 {
err = errors.New("KEY已存在")
return
}
p.redis.Set1(msg.Key(), goutils.M{
"topic": msg.Topic(),
"body": msg,
"headers": msg.Headers(),
"timestamp": time.Now().Format("2006-01-02 15:04:05"),
}.String(), time.Hour)
}
var producer sarama.SyncProducer
producer, err = sarama.NewSyncProducerFromClient(p.Client())
if err != nil {
return
}
defer producer.Close()
partition, offset, err = producer.SendMessage(m)
return
}
// 发送消息 - 异步 - 批量
func (p *producer) SendAsyncMessages(msgs []IMessage, cb MessageHandler) (err error) {
if len(msgs) == 0 {
return nil
}
var sendMsgs []*sarama.ProducerMessage
for _, msg := range msgs {
m := &sarama.ProducerMessage{
Topic: msg.Topic(),
Value: sarama.ByteEncoder(msg.Serialize()),
}
if v := msg.Key(); v != "" {
m.Key = sarama.StringEncoder(v)
}
if data := msg.Headers(); data != nil {
var headers []sarama.RecordHeader
for k, v := range data {
headers = append(headers, sarama.RecordHeader{
Key: []byte(k),
Value: []byte(v),
})
}
m.Headers = headers
}
sendMsgs = append(sendMsgs, m)
}
defer func() {
log := golog.WithTag("gokafka-producer").WithField("msg", goutils.M{
"send_msgs_length": len(sendMsgs),
})
if err != nil {
log.Error("消息发送失败", err)
return
}
//log.Debug("消息发送成功")
}()
producer, err := sarama.NewAsyncProducerFromClient(p.Client())
if err != nil {
return
}
defer producer.Close()
for _, msg := range sendMsgs {
producer.Input() <- msg
}
for i := 0; i < len(sendMsgs); i++ {
select {
case _msg := <-producer.Successes():
cb(&ProducerMessage{_msg}, nil)
case e := <-producer.Errors():
err = e.Err
cb(&ProducerMessage{e.Msg}, e.Err)
}
}
return
}
// 发送消息 - 异步
func (p *producer) SendAsyncMessage(msg IMessage, cb MessageHandler) (err error) {
m := &sarama.ProducerMessage{
Topic: msg.Topic(),
Value: sarama.ByteEncoder(msg.Serialize()),
}
if v := msg.Key(); v != "" {
m.Key = sarama.StringEncoder(v)
}
if data := msg.Headers(); data != nil {
var headers []sarama.RecordHeader
for k, v := range data {
headers = append(headers, sarama.RecordHeader{
Key: []byte(k),
Value: []byte(v),
})
}
m.Headers = headers
}
defer func() {
log := golog.WithTag("gokafka-producer").WithField("msg", goutils.M{
"topic": msg.Topic(),
"key": msg.Key(),
"headers": msg.Headers(),
"body": msg,
"partition": m.Partition,
"offset": m.Offset,
})
if err != nil {
log.Error("消息发送失败", err)
return
}
//log.Debug("消息发送成功")
}()
// 添加缓存
if p.redis != nil && len(msg.Key()) > 0 {
if p.focus {
p.redis.Del(msg.Key())
}
if p.redis.Exists(msg.Key()).Val() > 0 {
err = errors.New("KEY已存在")
return
}
p.redis.Set1(msg.Key(), goutils.M{
"topic": msg.Topic(),
"body": msg,
"headers": msg.Headers(),
"timestamp": time.Now().Format("2006-01-02 15:04:05"),
}.String(), time.Hour)
}
var producer sarama.AsyncProducer
producer, err = sarama.NewAsyncProducerFromClient(p.Client())
if err != nil {
return
}
defer producer.Close()
producer.Input() <- m
select {
case msg := <-producer.Successes():
cb(&ProducerMessage{msg}, nil)
case e := <-producer.Errors():
err = e.Err
cb(&ProducerMessage{e.Msg}, e.Err)
}
return
}
go-mq/go-kafka/client.go (7.3 KiB)
package gokafka
import (
"fmt"
"github.com/IBM/sarama"
goredis "github.com/gif-gif/go.io/go-db/go-redis"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/samber/lo"
"os"
"time"
)
type GoKafka struct {
conf Config
sarama.Client
redis *goredis.GoRedis
ConsumerGroupInstanceId string
}
func (cli *GoKafka) GetConfig() Config {
return cli.conf
}
func (cli *GoKafka) init() (err error) {
createUniqueInstanceId := func() string {
hostname, _ := os.Hostname()
timestamp := time.Now().Unix()
return fmt.Sprintf("consumer-%s-%d", hostname, timestamp)
}
//id := strconv.Itoa(os.Getpid())
id := createUniqueInstanceId()
cli.ConsumerGroupInstanceId = id
config := sarama.NewConfig()
config.ClientID = id
config.ChannelBufferSize = 1024
if cli.conf.Version == "" {
config.Version = sarama.V3_6_0_0
}
if cli.conf.User != "" {
config.Net.SASL.Enable = true
config.Net.SASL.User = cli.conf.User
config.Net.SASL.Password = cli.conf.Password
}
if cli.conf.KeepAlive == 0 {
cli.conf.KeepAlive = 10
}
config.Net.KeepAlive = time.Duration(cli.conf.KeepAlive) * time.Second
// 等所有follower都成功后再返回
config.Producer.RequiredAcks = sarama.WaitForAll
// 分区策略为Manual,指定分区发送消息
//config.Producer.Partitioner = sarama.NewManualPartitioner
// 分区策略为Hash,解决相同key的消息落在一个分区
//config.Producer.Partitioner = sarama.NewHashPartitioner
// 分区策略为Random,解决消费组分布式部署
config.Producer.Partitioner = sarama.NewRandomPartitioner
config.Producer.Return.Successes = true
config.Producer.Return.Errors = true
config.Consumer.Return.Errors = true
//批量发送策略
config.Producer.Flush.Messages = 100 // 积累100条消息
config.Producer.Flush.Bytes = 1048576 // 积累1MB数据
config.Producer.Flush.Frequency = 100 * time.Millisecond // 每100ms刷新
config.Consumer.Offsets.AutoCommit.Enable = true // 自动提交
config.Consumer.Offsets.AutoCommit.Interval = 1 * time.Second // 间隔
config.Consumer.Offsets.Retry.Max = 5
if cli.conf.OffsetNewest {
config.Consumer.Offsets.Initial = sarama.OffsetNewest
} else {
config.Consumer.Offsets.Initial = sarama.OffsetOldest
}
config.Consumer.Group.Rebalance.GroupStrategies = []sarama.BalanceStrategy{
sarama.NewBalanceStrategyRoundRobin(),
sarama.NewBalanceStrategySticky(),
sarama.NewBalanceStrategyRange(),
}
config.Consumer.Group.Heartbeat.Interval = 5 * time.Second
config.Consumer.Group.Session.Timeout = 15 * time.Second
config.Consumer.Group.Rebalance.Timeout = 12 * time.Second
config.Producer.Timeout = 10 * time.Second
if cli.conf.Timeout > 0 {
config.Producer.Timeout = time.Duration(cli.conf.Timeout) * time.Second
}
if cli.conf.HeartbeatInterval > 0 {
config.Consumer.Group.Heartbeat.Interval = time.Duration(cli.conf.HeartbeatInterval) * time.Second
}
if cli.conf.SessionTimeout > 0 {
config.Consumer.Group.Session.Timeout = time.Duration(cli.conf.SessionTimeout) * time.Second
}
if cli.conf.RebalanceTimeout > 0 {
config.Consumer.Group.Rebalance.Timeout = time.Duration(cli.conf.RebalanceTimeout) * time.Second
}
config.Consumer.Group.InstanceId = id
cli.Client, err = sarama.NewClient(cli.conf.Addrs, config)
if err != nil {
golog.WithTag("gokafka").Error(err)
}
if cfg := cli.conf.RedisConfig; cfg.Addr != "" {
cli.redis, err = goredis.New(cfg)
if err != nil {
golog.WithTag("gokafka").Error("Redis 初始化失败", err)
}
}
return
}
//func (cli *GoKafka) CreateTopics(topics []string) (err error) {
// kafkaTopics, err := Producer().Client().Topics()
// if err != nil {
// golog.WithTag("kafka-producer").Error(err)
// }
// for _, topic := range topics {
// if lo.Contains(kafkaTopics, topic) {
// continue
// }
// msg := KafkaMessageTest{
// TopicName: topic,
// }
//
// _, _, err := gokafka.Producer().SendMessage(&msg)
// if err != nil {
// golog.Warn("KafkaMessageTest send message failed: ", err.Error())
// }
// fmt.Println("KafkaMessageTest send message success:" + topic)
// }
//}
func (cli *GoKafka) CreateTopicRequest(topicName string, partitions int, replicationFactors int) error {
kafkaTopics, err := Producer().Client().Topics()
if err != nil {
return err
}
if lo.Contains(kafkaTopics, topicName) {
return nil
}
request := &sarama.CreateTopicsRequest{}
request.TopicDetails = make(map[string]*sarama.TopicDetail)
request.TopicDetails[topicName] = &sarama.TopicDetail{
NumPartitions: int32(partitions),
ReplicationFactor: int16(replicationFactors),
}
broker := cli.Brokers()[0]
err = broker.Open(cli.Config())
if err != nil {
return err
}
defer broker.Close()
ok, err := broker.Connected()
if err != nil {
return err
}
if ok {
_, err = broker.CreateTopics(request)
return err
} else {
return fmt.Errorf(" broker is not connected")
}
}
func (cli *GoKafka) CreateTopicsRequest(topicNames []string, partitions int, replicationFactors int) error {
for _, topicName := range topicNames {
err := cli.CreateTopicRequest(topicName, partitions, replicationFactors)
if err != nil {
return err
}
}
return nil
}
func (cli *GoKafka) Close() {
if !cli.Client.Closed() {
cli.Client.Close()
}
}
// 消费者
func (cli *GoKafka) Consumer() IConsumer {
return &consumer{GoKafka: cli}
}
// 生产者
func (cli *GoKafka) Producer(opts ...Option) IProducer {
var focus bool
for _, opt := range opts {
switch opt.Name {
case FocusName:
focus = opt.Value.(bool)
}
}
return &producer{GoKafka: cli, focus: focus}
}
func (c *GoKafka) GetKey(topic, msg string) string {
return fmt.Sprintf("goio:mq:%s:%s", time.Now().Format("20060102"), goutils.MD5([]byte(topic+msg)))
}
func (c *GoKafka) Redis() *goredis.GoRedis {
return c.redis
}
// 主题列表
func (c *GoKafka) Topics() []string {
if c.Client == nil {
return []string{}
}
topics, err := c.Client.Topics()
if err != nil {
golog.WithTag("gokafka").Error(err)
return []string{}
}
return topics
}
// 分区数量
func (c *GoKafka) Partitions(topic string) []int32 {
if c.Client == nil {
return []int32{}
}
partitions, err := c.Client.Partitions(topic)
if err != nil {
golog.WithTag("gokafka").WithField("topic", topic).Error(err)
return []int32{}
}
return partitions
}
// 分区数量
func (c *GoKafka) OffsetInfo(topic, groupId string) (data []map[string]int64) {
data = []map[string]int64{}
if c.Client == nil {
return
}
partitions := c.Partitions(topic)
if l := len(partitions); l == 0 {
return
}
var (
l = golog.WithTag("gokafka").WithField("groupId", groupId).WithField("topic", topic)
)
om, err := sarama.NewOffsetManagerFromClient(groupId, c.Client)
if err != nil {
l.Error(err)
return
}
defer om.Close()
for _, partition := range partitions {
offset, err := c.Client.GetOffset(topic, partition, -1)
if err != nil {
l.Error(err)
continue
}
pom, err := om.ManagePartition(topic, partition)
if err != nil {
l.Error(err)
continue
}
nextOffset, msg := pom.NextOffset()
if msg != "" {
l.Error(msg)
continue
}
backlog := offset
if nextOffset != -1 {
backlog -= nextOffset
}
data = append(data, map[string]int64{
"partition": int64(partition),
"offset": offset,
"nextOffset": nextOffset,
"backlog": backlog,
})
}
return
}
go-mq/go-kafka/config.go (1.1 KiB)
package gokafka
import goredis "github.com/gif-gif/go.io/go-db/go-redis"
type Config struct {
User string `json:"user,optional" yaml:"User"`
Name string `json:"name,optional" yaml:"Name"`
Password string `json:"password,optional" yaml:"Password"`
Addrs []string `json:"addrs,optional" yaml:"Addrs"`
Timeout int `json:"timeout,optional" yaml:"Timeout"` // 单位:秒
HeartbeatInterval int `json:"heartbeatInterval,optional" yaml:"HeartbeatInterval"` // 单位:秒
SessionTimeout int `json:"sessionTimeout,optional" yaml:"SessionTimeout"` // 单位:秒
RebalanceTimeout int `json:"rebalanceTimeout,optional" yaml:"RebalanceTimeout"` // 单位:秒
OffsetNewest bool `json:"offsetNewest,optional" yaml:"OffsetNewest"`
Version string `json:"version,optional" yaml:"Version"`
KeepAlive int64 `json:"keepAlive,optional" yaml:"KeepAlive"` // 单位:秒
GroupId string `json:"groupId,optional" yaml:"GroupId"`
RedisConfig goredis.Config `json:"redisConfig,optional" yaml:"RedisConfig"`
}
go-mq/go-kafka/kafka.go (1.6 KiB)
package gokafka
import (
"errors"
goredis "github.com/gif-gif/go.io/go-db/go-redis"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoKafka{}
func Init(conf Config, opts ...Option) (err error) {
if conf.Name == "" {
conf.Name = "default"
}
if __clients[conf.Name] != nil {
return errors.New("GoKafka already exists")
}
__clients[conf.Name], err = New(conf, opts...)
if err != nil {
return err
}
return nil
}
func New(conf Config, opts ...Option) (*GoKafka, error) {
__client := &GoKafka{conf: conf}
//goutils.AsyncFunc(func() {
// select {
// case <-gocontext.Cancel().Done():
// __client.Close()
// return
// }
//})
err := __client.init()
for _, opt := range opts {
switch opt.Name {
case RedisName:
__client.redis = opt.Value.(*goredis.GoRedis)
}
}
return __client, err
}
func GetClient(names ...string) *GoKafka {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
// default or 只有一个kafka实例直接返回
func Client() *GoKafka {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("gokafka").Error("no default kafka GoKafka")
return nil
}
func Consumer() IConsumer {
return GetClient().Consumer()
}
func Producer(opts ...Option) IProducer {
return GetClient().Producer(opts...)
}
go-mq/go-kafka/option.go (387 B)
package gokafka
import goredis "github.com/gif-gif/go.io/go-db/go-redis"
const (
FocusName = "focus"
RedisName = "redis"
)
type Option struct {
Name string
Value interface{}
}
// 是否强制
func FocusOption() Option {
return Option{Name: FocusName, Value: true}
}
// redis 对象
func RedisOption(cli *goredis.GoRedis) Option {
return Option{Name: RedisName, Value: cli}
}
go-mq/go-kafka/readme.md (1.4 KiB)
kafka
- https://github.com/IBM/sarama
gokafka.Client().Topics()
// 发送消息,不指定分区
gokafka.Producer().SendMessage("test", []byte("hi"))
// 发送消息,指定分区
gokafka.Producer().WithPartition(0).SendMessage("test", []byte("hi goio"))
// 发送异步消息,不指定分区
gokafka.Producer().SendAsyncMessage("test", []byte("hi goio"), func(msg *gokafka.ProducerMessage, err error) {
})
// 发送异步消息,指定分区
gokafka.Producer().WithPartition(0).SendAsyncMessage("test", []byte("hi goio"), func(msg *gokafka.ProducerMessage, err error) {
})
// 消费消息,指定分区,指定起始位置
gokafka.Consumer().WithPartition(0).WithOffset(100).Consume("test", func(msg *gokafka.ConsumerMessage, consumerErr *gokafka.ConsumerError) error {
return nil
})
// 消费消息,指定分区,从最新位置开始
gokafka.Consumer().WithPartition(0).WithOffsetNewest().Consume("test", func(msg *gokafka.ConsumerMessage, consumerErr *gokafka.ConsumerError) error {
return nil
})
// 消费消息,指定分区,从最头开始
gokafka.Consumer().WithPartition(0).WithOffsetOldest().Consume("test", func(msg *gokafka.ConsumerMessage, consumerErr *gokafka.ConsumerError) error {
return nil
})
// 消费消息,分组消息,分组里面只要1个消费者消费
gokafka.Consumer().ConsumeGroup("test-id", []string{"test"}, func(msg *gokafka.ConsumerMessage, consumerErr *gokafka.ConsumerError) error {
return nil
})
go-mq/go-kafka/type.go (1.4 KiB)
package gokafka
import (
"encoding/json"
"fmt"
"github.com/IBM/sarama"
gocontext "github.com/gif-gif/go.io/go-context"
"github.com/samber/lo"
)
type MessageHandler func(msg *ProducerMessage, err error)
type ProducerMessage struct {
*sarama.ProducerMessage
}
type ConsumerHandler func(ctx *gocontext.Context, msg *ConsumerMessage, consumerErr *ConsumerError) error
type ConsumerMessage struct {
*sarama.ConsumerMessage
GroupSession sarama.ConsumerGroupSession
}
func (msg ConsumerMessage) Commit() {
if msg.GroupSession == nil {
return
}
msg.GroupSession.MarkMessage(msg.ConsumerMessage, "")
}
type ConsumerError struct {
*sarama.ConsumerError
}
func (t *KafkaMsg) Topic() string {
return t.KafkaTopic
}
func (t *KafkaMsg) Key() string {
return lo.If(t.KafkaTopic != "", t.KafkaTopic).Else(t.KafkaKey)
}
func (t *KafkaMsg) Headers() map[string]string {
return lo.If(t.KafkaHeaders != nil, t.KafkaHeaders).Else(map[string]string{})
}
func (t *KafkaMsg) Serialize() []byte {
b, _ := json.Marshal(t.Data)
return b
}
func (t *KafkaMsg) Deserialize(b []byte) {
if err := json.Unmarshal(b, &t.Data); err != nil {
fmt.Println(err)
}
}
type KafkaMsg struct {
KafkaTopic string `json:"kafkaTopic"`
KafkaKey string `json:"kafkaKey"`
KafkaHeaders map[string]string `json:"kafkaHeaders"`
Data interface{} `json:"data"`
}
go-mq/go-mqtt/client.go (4.1 KiB)
package gomqtt
import (
"crypto/tls"
"crypto/x509"
"fmt"
mqtt "github.com/eclipse/paho.mqtt.golang"
"io/ioutil"
"log"
"math/rand"
"time"
)
const (
PROTOCOL_TCP = "tcp"
PROTOCOL_SSL = "ssl"
PROTOCOL_WS = "ws"
PROTOCOL_WSS = "wss"
)
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
Protocol string `yaml:"Protocol" json:"protocol,optional"`
CaFilePath string `yaml:"CaFilePath" json:"caFilePath,optional"` //tls(ssl wss) 证书位置
Server string `yaml:"Server" json:"server,optional"`
Port int `yaml:"Port" json:"port,optional"`
ClientId string `yaml:"ClientId" json:"clientId,optional"`
User string `yaml:"User" json:"user,optional"`
Secret string `yaml:"Secret" json:"secret,optional"`
KeepAlive int64 `yaml:"KeepAlive" json:"keepAlive,optional"` //s
AutoReconnect bool `yaml:"AutoReconnect" json:"autoReconnect,optional"`
ConnectRetry bool `yaml:"ConnectRetry" json:"connectRetry,optional"`
ConnectTimeout int64 `yaml:"ConnectTimeout" json:"connectTimeout,optional"`
DefaultHandler *mqtt.MessageHandler
OnConnect *mqtt.OnConnectHandler
OnConnectionLost *mqtt.ConnectionLostHandler
}
type GoMqttClient struct {
Client mqtt.Client
}
func NewClient(config Config) (*GoMqttClient, error) {
if config.ConnectTimeout <= 0 {
config.ConnectTimeout = 10
}
if config.KeepAlive <= 0 {
config.KeepAlive = 60 * 2
}
if config.Protocol == "" {
config.Protocol = "tcp"
}
opts := mqtt.NewClientOptions()
opts.AddBroker(fmt.Sprintf("%s://%s:%d/mqtt", config.Protocol, config.Server, config.Port))
if config.ClientId == "" {
rand.Seed(time.Now().UnixNano())
config.ClientId = fmt.Sprintf("go-client-%d", rand.Int())
}
opts.SetClientID(config.ClientId)
opts.SetUsername(config.User)
opts.SetPassword(config.Secret)
opts.SetAutoReconnect(config.AutoReconnect)
opts.SetConnectRetry(config.ConnectRetry)
opts.SetKeepAlive(time.Duration(config.KeepAlive) * time.Second)
opts.SetConnectTimeout(time.Duration(config.ConnectTimeout) * time.Second)
// Optional: 设置CA证书
if config.CaFilePath != "" {
opts.SetTLSConfig(loadTLSConfig(config.CaFilePath))
}
if config.DefaultHandler != nil {
opts.SetDefaultPublishHandler(*config.DefaultHandler)
}
if config.OnConnectionLost != nil {
opts.SetConnectionLostHandler(*config.OnConnectionLost)
}
if config.OnConnect != nil {
opts.SetOnConnectHandler(*config.OnConnect)
}
gc, err := NewConfig(opts)
if err != nil {
return nil, err
}
return gc, nil
}
func NewConfig(opts *mqtt.ClientOptions) (*GoMqttClient, error) {
gomqtt := &GoMqttClient{}
client := mqtt.NewClient(opts)
if token := client.Connect(); token.Wait() && token.Error() != nil {
return nil, token.Error()
}
gomqtt.Client = client
return gomqtt, nil
}
func loadTLSConfig(caFile string) *tls.Config {
// load tls config
var tlsConfig tls.Config
tlsConfig.InsecureSkipVerify = false
if caFile != "" {
certpool := x509.NewCertPool()
ca, err := ioutil.ReadFile(caFile)
if err != nil {
log.Fatal(err.Error())
}
certpool.AppendCertsFromPEM(ca)
tlsConfig.RootCAs = certpool
}
return &tlsConfig
}
func (g *GoMqttClient) Connect() error {
if token := g.Client.Connect(); token.Wait() && token.Error() != nil {
return token.Error()
}
return nil
}
func (g *GoMqttClient) Disconnect(quiesce uint) {
g.Client.Disconnect(quiesce)
}
func (g *GoMqttClient) Subscribe(topic string, qos byte, callback mqtt.MessageHandler) error {
if token := g.Client.Subscribe(topic, qos, callback); token.Wait() && token.Error() != nil {
return token.Error()
}
return nil
}
func (g *GoMqttClient) Publish(topic string, qos byte, retained bool, payload interface{}) error {
if token := g.Client.Publish(topic, qos, retained, payload); token.Wait() && token.Error() != nil {
return token.Error()
}
return nil
}
func (g *GoMqttClient) Unsubscribe(topics ...string) error {
if token := g.Client.Unsubscribe(topics...); token.Wait() && token.Error() != nil {
return token.Error()
}
return nil
}
func (g *GoMqttClient) isConnected() bool {
return g.Client.IsConnected()
}
go-mq/go-mqtt/gomqtt.go (1.0 KiB)
package gomqtt
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoMqttClient{}
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" || name == "default" {
conf.Name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name], err = NewClient(conf)
if err != nil {
return err
}
}
return nil
}
func GetClient(names ...string) *GoMqttClient {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Client() *GoMqttClient {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("gomqtt").Error("no default mqtt client")
return nil
}
go-mq/go-mqtt/readme.md (389 B)
MQTT Go client
- https://github.com/eclipse/paho.mqtt.golang
mqtt server 事件中心
- https://github.com/emqx/emqx erlang server
-
https://github.com/mochi-mqtt golang server
-
${EMQX_TCP_PORT}:1883
- ${EMQX_SSL_PORT}:8883
- ${EMQX_WS_PORT}:8083
- ${EMQX_WSS_PORT}:8084
- ${EMQX_DASHBOARD_PORT}:18083
注意
- 每个客户端都需要单独的 ClientId ,不能重复
go-mq/go-mqtt/test/main.go (2.7 KiB)
package main
import (
"fmt"
mqtt "github.com/eclipse/paho.mqtt.golang"
gocontext "github.com/gif-gif/go.io/go-context"
golog "github.com/gif-gif/go.io/go-log"
gomqtt "github.com/gif-gif/go.io/go-mq/go-mqtt"
goutils "github.com/gif-gif/go.io/go-utils"
"time"
)
var topic = "topic/test"
func main() {
testSubscribe()
time.Sleep(time.Second * 2)
<-gocontext.WithCancel().Done()
}
func testSubscribe() {
var messagePubHandler mqtt.MessageHandler = func(client mqtt.Client, msg mqtt.Message) {
fmt.Printf("testSubscribe Received message: %s from topic: %s\n", msg.Payload(), msg.Topic())
}
var connectHandler mqtt.OnConnectHandler = func(client mqtt.Client) {
fmt.Println("testSubscribe Connected")
goutils.AsyncFunc(func() {
testPublish()
})
}
var connectLostHandler mqtt.ConnectionLostHandler = func(client mqtt.Client, err error) {
fmt.Printf("testSubscribe Connect lost: %v", err)
}
var conf = gomqtt.Config{
Server: "127.0.0.1",
Port: 18830,
ClientId: "go_mqtt_client_test",
User: "mqtt",
Secret: "223238",
DefaultHandler: &messagePubHandler,
OnConnect: &connectHandler,
OnConnectionLost: &connectLostHandler,
}
conf.Name = "testSubscribe"
err := gomqtt.Init(conf)
if err != nil {
golog.Error(err.Error())
return
}
c := gomqtt.GetClient(conf.Name)
c.Client.Unsubscribe(topic)
err = c.Subscribe(topic, 1, func(c mqtt.Client, msg mqtt.Message) {
golog.WithTag("mqtt-Subscribe").Info(string(msg.Payload()))
})
if err != nil {
golog.Error(err.Error())
return
}
}
func testPublish() {
var messagePubHandler mqtt.MessageHandler = func(client mqtt.Client, msg mqtt.Message) {
fmt.Printf("testPublish Received message: %s from topic: %s\n", msg.Payload(), msg.Topic())
}
var connectHandler mqtt.OnConnectHandler = func(client mqtt.Client) {
fmt.Println("testPublish Connected")
}
var connectLostHandler mqtt.ConnectionLostHandler = func(client mqtt.Client, err error) {
fmt.Printf("testPublish Connect lost: %v", err)
}
var conf = gomqtt.Config{
Server: "127.0.0.1",
Port: 18830,
ClientId: "go_mqtt_client_test1",
User: "mqtt",
Secret: "223238",
DefaultHandler: &messagePubHandler,
OnConnect: &connectHandler,
OnConnectionLost: &connectLostHandler,
}
conf.Name = "testPublish"
err := gomqtt.Init(conf)
if err != nil {
golog.Error(err.Error())
return
}
publish(gomqtt.GetClient(conf.Name))
<-gocontext.WithCancel().Done()
}
func publish(client *gomqtt.GoMqttClient) {
err := client.Publish(topic, 0, true, "publish-text")
if err != nil {
golog.Error(err.Error())
return
}
golog.WithTag("mqtt-publish").WithField(topic, topic).Info("publish-text")
}
go-mq/go-rabbitmq/readme.md (12 B)
rabbitmq
go-mq/go-rocketmq/readme.md (50 B)
RocketMQ
- github.com/apache/rocketmq-client-go
go-mq/go-rocketmq/test/admin/main.go (2.7 KiB)
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"context"
"fmt"
"github.com/apache/rocketmq-client-go/v2/admin"
"github.com/apache/rocketmq-client-go/v2/primitive"
"time"
)
func main() {
//testTopic()
testGroup()
}
func testTopic() {
topic := "newOne"
//clusterName := "DefaultCluster"
nameSrvAddr := []string{"122.228.113.238:9876"}
brokerAddr := "122.228.113.238:10911"
testAdmin, err := admin.NewAdmin(
admin.WithResolver(primitive.NewPassthroughResolver(nameSrvAddr)),
admin.WithCredentials(primitive.Credentials{
AccessKey: "RocketMQ",
SecretKey: "12345678",
}),
)
// topic list
result, err := testAdmin.FetchAllTopicList(context.Background())
if err != nil {
fmt.Println("FetchAllTopicList error:", err.Error())
}
fmt.Println(result.TopicList)
//create topic
err = testAdmin.CreateTopic(
context.Background(),
admin.WithTopicCreate(topic),
admin.WithBrokerAddrCreate(brokerAddr),
)
if err != nil {
fmt.Println("Create topic error:", err.Error())
}
//deletetopic
err = testAdmin.DeleteTopic(
context.Background(),
admin.WithTopicDelete(topic),
//admin.WithBrokerAddrDelete(brokerAddr),
//admin.WithNameSrvAddr(nameSrvAddr),
)
if err != nil {
fmt.Println("Delete topic error:", err.Error())
}
err = testAdmin.Close()
if err != nil {
fmt.Printf("Shutdown admin error: %s", err.Error())
}
}
func testGroup() {
//clusterName := "DefaultCluster"
nameSrvAddr := []string{"122.228.113.238:9876"}
brokerAddr := "122.228.113.238:10911"
testAdmin, err := admin.NewAdmin(
admin.WithResolver(primitive.NewPassthroughResolver(nameSrvAddr)),
admin.WithCredentials(primitive.Credentials{
AccessKey: "RocketMQ",
SecretKey: "12345678",
}),
)
// group list
result, err := testAdmin.GetAllSubscriptionGroup(context.Background(), brokerAddr, 3*time.Second)
if err != nil {
fmt.Println("GetAllSubscriptionGroup error:", err.Error())
}
fmt.Println(result.SubscriptionGroupTable)
err = testAdmin.Close()
if err != nil {
fmt.Printf("Shutdown admin error: %s", err.Error())
}
}
go-mq/gomq.go (2.3 KiB)
package gomq
import (
"errors"
"sync"
"time"
)
type BlockingQueue struct {
queue chan interface{}
wg sync.WaitGroup
closed bool
closeMux sync.RWMutex
}
// NewBlockingQueue 创建一个新的阻塞队列
func NewBlockingQueue(size int) *BlockingQueue {
return &BlockingQueue{
queue: make(chan interface{}, size),
closed: false,
}
}
// Enqueue 将元素添加到队列
func (bq *BlockingQueue) Enqueue(item interface{}) error {
bq.closeMux.RLock()
if bq.closed {
bq.closeMux.RUnlock()
return errors.New("queue is closed")
}
bq.closeMux.RUnlock()
// 使用 select 防止死锁
select {
case bq.queue <- item:
// 如果成功添加元素,增加等待组计数
bq.wg.Add(1)
return nil
default:
return errors.New("queue is full")
}
}
// EnqueueBlocking 阻塞式将元素添加到队列
func (bq *BlockingQueue) EnqueueBlocking(item interface{}) error {
bq.closeMux.RLock()
if bq.closed {
bq.closeMux.RUnlock()
return errors.New("queue is closed")
}
bq.wg.Add(1)
bq.closeMux.RUnlock()
// 此处会阻塞直到队列有空间
select {
case bq.queue <- item:
return nil
}
}
// Dequeue 从队列中移除并返回一个元素,如果队列为空则阻塞
func (bq *BlockingQueue) Dequeue() (interface{}, error) {
item, ok := <-bq.queue
if !ok {
return nil, errors.New("queue is closed")
}
bq.wg.Done()
return item, nil
}
// DequeueWithTimeout 带超时的出队操作
func (bq *BlockingQueue) DequeueWithTimeout(timeout time.Duration) (interface{}, error) {
select {
case item, ok := <-bq.queue:
if !ok {
return nil, errors.New("queue is closed")
}
bq.wg.Done()
return item, nil
case <-time.After(timeout):
return nil, errors.New("dequeue timeout")
}
}
// Size 返回队列中的元素数量(注意:这只是一个瞬时值)
func (bq *BlockingQueue) Size() int {
return len(bq.queue)
}
// Wait 等待所有元素被处理
func (bq *BlockingQueue) Wait() {
bq.wg.Wait()
}
// Close 关闭队列,不再接受新元素
func (bq *BlockingQueue) Close() {
bq.closeMux.Lock()
defer bq.closeMux.Unlock()
if !bq.closed {
bq.closed = true
close(bq.queue)
}
}
// IsClosed 检查队列是否已关闭
func (bq *BlockingQueue) IsClosed() bool {
bq.closeMux.RLock()
defer bq.closeMux.RUnlock()
return bq.closed
}
go-mq/readme.md (29 B)
消息中间件操作 MQ
go-oss/go-alioss/ali-oss.go (1.6 KiB)
package goalioss
import (
"errors"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*Uploader{}
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
__clients[name], err = create(conf)
if err != nil {
return
}
}
return
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func create(conf Config) (*Uploader, error) {
o := &Uploader{
conf: conf,
options: []oss.Option{},
}
client, err := o.getClient()
if err != nil {
golog.Error(err.Error())
return nil, err
}
o.client = client
bucket, err := o.getBucket()
if err != nil {
golog.Error(err.Error())
return nil, err
}
o.bucket = bucket
return o, nil
}
func New(conf Config) (*Uploader, error) {
err := Init(conf)
if err != nil {
return nil, err
}
return GetClient(conf.Name), nil
}
func GetClient(names ...string) *Uploader {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func Default() *Uploader {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("gominio").Error("no default minio client")
return nil
}
func (g *Uploader) AliClient() *oss.Client {
return g.client
}
go-oss/go-alioss/config.go (513 B)
package goalioss
type Config struct {
Name string `json:"name,optional" yaml:"Name"`
AccessKeyId string `json:"accessKeyId,optional" yaml:"AccessKeyId"`
AccessKeySecret string `json:"accessKeySecret,optional" yaml:"AccessKeySecret"`
Endpoint string `json:"endpoint,optional" yaml:"Endpoint"`
Bucket string `json:"bucket,optional" yaml:"Bucket"`
Domain string `json:"domain,optional" yaml:"Domain"`
Open bool `json:"open,optional" yaml:"Open"`
}
go-oss/go-alioss/readme.md (77 B)
Aliyun oss 封装
基于库 github.com/aliyun/aliyun-oss-go-sdk/oss
go-oss/go-alioss/test/main.go (1.6 KiB)
package main
import (
"flag"
"fmt"
goalioss "github.com/gif-gif/go.io/go-oss/go-alioss"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gif-gif/go.io/goio"
"io/ioutil"
"os"
"strings"
)
var (
AccessKeyId = flag.String("access_key_id", "", "")
AccessKeySecret = flag.String("access_key_secret", "", "")
Endpoint = flag.String("endpoint", "", "")
Bucket = flag.String("bucket", "", "")
Domain = flag.String("domain", "", "")
)
func main() {
goio.FlagInit()
args := os.Args
if l := len(args); l < 2 {
fmt.Println("请选择上传文件!")
return
}
conf := goalioss.Config{
AccessKeyId: *AccessKeyId,
AccessKeySecret: *AccessKeySecret,
Endpoint: *Endpoint,
Bucket: *Bucket,
Domain: *Domain,
}
if conf.AccessKeyId == "" {
conf.AccessKeyId = ""
}
if conf.AccessKeySecret == "" {
conf.AccessKeySecret = ""
}
if conf.Endpoint == "" {
conf.Endpoint = "oss-cn-beijing.aliyuncs.com"
}
if conf.Bucket == "" {
conf.Bucket = ""
}
err := goalioss.Init(conf)
if err != nil {
return
}
for n, i := range args {
if n == 0 {
continue
}
b, err := ioutil.ReadFile(i)
if err != nil {
fmt.Println(err.Error())
return
}
index := strings.LastIndex(i, "/")
var filename string
if index == -1 {
filename = i
} else {
filename = i[index+1:]
}
md5 := goutils.MD5(b)
filename = fmt.Sprintf("%s/%s/%s", md5[0:2], md5[2:4], filename)
url, err := goalioss.Default().Upload(strings.ToLower(filename), b)
if err != nil {
fmt.Println(err.Error())
continue
}
fmt.Println(url)
}
}
go-oss/go-alioss/uploader.go (1.5 KiB)
package goalioss
import (
"bytes"
"errors"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"path"
"strings"
)
type Uploader struct {
conf Config
client *oss.Client
bucket *oss.Bucket
options []oss.Option
}
func (o *Uploader) ContentType(value string) *Uploader {
o.options = append(o.options, oss.ContentType(value))
return o
}
func (o *Uploader) Options(opts ...oss.Option) *Uploader {
o.options = append(o.options, opts...)
return o
}
func (o *Uploader) Upload(filename string, body []byte) (string, error) {
if filename == "" {
return "", errors.New("文件名为空")
}
md5str := goutils.MD5(body)
ext := path.Ext(filename)
index := strings.Index(filename, ext)
filename = filename[:index] + "_" + md5str[8:24] + filename[index:]
if err := o.bucket.PutObject(filename, bytes.NewReader(body), o.options...); err != nil {
golog.Error(err.Error())
return "", err
}
if filename[0:1] != "/" {
filename = "/" + filename
}
if o.conf.Domain != "" {
if idx, l := strings.LastIndex(o.conf.Domain, "/"), len(o.conf.Domain); idx+1 == l {
o.conf.Domain = o.conf.Domain[:l-1]
}
return o.conf.Domain + filename, nil
}
url := "https://" + o.conf.Bucket + "." + o.conf.Endpoint + filename
return url, nil
}
func (o *Uploader) getClient() (*oss.Client, error) {
return oss.New(o.conf.Endpoint, o.conf.AccessKeyId, o.conf.AccessKeySecret)
}
func (o *Uploader) getBucket() (*oss.Bucket, error) {
return o.client.Bucket(o.conf.Bucket)
}
go-oss/go-minio/README.md (3.0 KiB)
go.minio
基于 github.com/minio/minio-go/v7 封装
package main
import (
"context"
"flag"
"fmt"
gominio "github.com/gif-gif/go.io/go-minio"
"github.com/minio/minio-go/v7"
"net/url"
"os"
"strings"
"time"
)
var (
AccessKeyId = flag.String("access_key_id", "fPztco0yxWC1qrxz6iDN", "")
AccessKeySecret = flag.String("access_key_secret", "kWS6q1PIxjcEnLSkoBWrWqPnsHFHm1Q8cvG1CUPm", "")
Endpoint = flag.String("endpoint", "minio.gif00.com", "")
Bucket = flag.String("bucket", "test", "")
Domain = flag.String("domain", "", "")
)
func createBucketTest() {
conf := gominio.Config{
AccessKeyId: *AccessKeyId,
AccessKeySecret: *AccessKeySecret,
Endpoint: *Endpoint,
Bucket: *Bucket,
Domain: *Domain,
UseSSL: false,
}
oss := gominio.New(conf)
bucketName := "testbucket"
location := "us-east-1"
err := oss.CreateBucket(context.Background(), bucketName, location)
if err != nil {
fmt.Println(err.Error())
} else {
fmt.Println("Created bucket successfully")
}
}
func uploadTest() {
args := os.Args
if l := len(args); l < 2 {
fmt.Println("请选择上传文件!")
return
}
conf := gominio.Config{
AccessKeyId: *AccessKeyId,
AccessKeySecret: *AccessKeySecret,
Endpoint: *Endpoint,
Bucket: *Bucket,
Domain: *Domain,
UseSSL: false,
}
oss := gominio.New(conf)
for n, i := range args {
if n == 0 {
continue
}
var filename string
index := strings.LastIndex(i, "/")
if index == -1 {
filename = i
} else {
filename = i[index+1:]
}
info, err := oss.FPutObject(context.Background(), strings.ToLower(filename), i, &minio.PutObjectOptions{
Expires: time.Now().AddDate(0, 0, 1),
})
if err != nil {
fmt.Println(err.Error())
continue
}
fmt.Println(info)
}
}
func getTest() {
conf := gominio.Config{
AccessKeyId: *AccessKeyId,
AccessKeySecret: *AccessKeySecret,
Endpoint: *Endpoint,
Bucket: *Bucket,
Domain: *Domain,
UseSSL: false,
}
oss := gominio.New(conf)
err := oss.FGetObject(context.Background(), "/test/2024/05/422744271b108960a4818cc91a1822d9.log", "/Users/Jerry/Desktop/bak202405/422744271b108960a4818cc91a1822d9.log", nil)
if err != nil {
fmt.Println(err.Error())
} else {
fmt.Println("get object succeeded")
}
for object := range oss.ListObjects("test", nil) {
if object.Err != nil {
fmt.Println(object.Err)
return
}
fmt.Println("found:", object)
}
// Set request parameters for content-disposition.
reqParams := make(url.Values)
reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"")
// Generates a presigned url which expires in a day.
presignedURL, err := oss.PresignedGetObject("test", "test.apk", time.Second*24*60*60, reqParams)
if err != nil {
fmt.Println(err)
return
}
fmt.Println("Successfully generated presigned URL", presignedURL)
}
func main() {
//uploadTest()
//createBucketTest()
getTest()
}
go-oss/go-minio/config.go (611 B)
package gominio
type Config struct {
Name string `json:"name,optional" yaml:"Name"`
AccessKeyId string `json:"accessKeyId,optional" yaml:"AccessKeyId"`
AccessKeySecret string `json:"accessKeySecret,optional" yaml:"AccessKeySecret"`
Endpoint string `json:"endpoint,optional" yaml:"Endpoint"`
//Bucket string `json:"bucket,optional" yaml:"Bucket"`
//Dir string `json:"dir,optional" yaml:"Dir"`
Domain string `json:"domain,optional" yaml:"Domain,optional"`
UseSSL bool `json:"useSSL,optional" yaml:"UseSSL"`
Open bool `json:"open,optional" yaml:"Open"`
}
go-oss/go-minio/gominio.go (1.2 KiB)
package gominio
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*Uploader{}
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("client already exists")
}
o, err := Create(conf)
if err != nil {
return err
}
__clients[name] = o
}
return
}
func New(conf Config) (*Uploader, error) {
err := Init(conf)
if err != nil {
return nil, err
}
return GetClient(conf.Name), nil
}
func GetClient(names ...string) *Uploader {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *Uploader {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("gominio").Error("no default minio client")
return nil
}
go-oss/go-minio/test/main.go (3.0 KiB)
package main
import (
"context"
"flag"
"fmt"
gominio "github.com/gif-gif/go.io/go-oss/go-minio"
"github.com/minio/minio-go/v7"
"net/url"
"os"
"strings"
"time"
)
var (
AccessKeyId = flag.String("access_key_id", "fPztco0yxWC1qrxz6iDN", "")
AccessKeySecret = flag.String("access_key_secret", "kWS6q1PIxjcEnLSkoBWrWqPnsHFHm1Q8cvG1CUPm", "")
Endpoint = flag.String("endpoint", "minio.gif00.com", "")
Bucket = flag.String("bucket", "test", "")
Domain = flag.String("domain", "", "")
)
func createBucketTest() {
conf := gominio.Config{
AccessKeyId: *AccessKeyId,
AccessKeySecret: *AccessKeySecret,
Endpoint: *Endpoint,
Domain: *Domain,
UseSSL: false,
}
oss, _ := gominio.Create(conf)
bucketName := "testbucket"
location := "us-east-1"
err := oss.CreateBucket(context.Background(), bucketName, location)
if err != nil {
fmt.Println(err.Error())
} else {
fmt.Println("Created bucket successfully")
}
}
func uploadTest() {
args := os.Args
if l := len(args); l < 2 {
fmt.Println("请选择上传文件!")
return
}
conf := gominio.Config{
AccessKeyId: *AccessKeyId,
AccessKeySecret: *AccessKeySecret,
Endpoint: *Endpoint,
Domain: *Domain,
UseSSL: false,
}
oss, _ := gominio.Create(conf)
for n, i := range args {
if n == 0 {
continue
}
var filename string
index := strings.LastIndex(i, "/")
if index == -1 {
filename = i
} else {
filename = i[index+1:]
}
info, err := oss.FPutObject(context.Background(), *Bucket, strings.ToLower(filename), i, &minio.PutObjectOptions{
Expires: time.Now().AddDate(0, 0, 1),
})
if err != nil {
fmt.Println(err.Error())
continue
}
fmt.Println(info)
}
}
func getTest() {
conf := gominio.Config{
AccessKeyId: *AccessKeyId,
AccessKeySecret: *AccessKeySecret,
Endpoint: *Endpoint,
Domain: *Domain,
UseSSL: false,
}
err := gominio.Init(conf)
if err != nil {
fmt.Println(err.Error())
return
}
err = gominio.Default().FGetObject(context.Background(), *Bucket, "/test/2024/05/422744271b108960a4818cc91a1822d9.log", "/Users/Jerry/Desktop/bak202405/422744271b108960a4818cc91a1822d9.log", nil)
if err != nil {
fmt.Println(err.Error())
return
} else {
fmt.Println("get object succeeded")
}
for object := range gominio.Default().ListObjects("test", nil) {
if object.Err != nil {
fmt.Println(object.Err)
return
}
fmt.Println("found:", object)
}
// Set request parameters for content-disposition.
reqParams := make(url.Values)
reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"")
// Generates a presigned url which expires in a day.
presignedURL, err := gominio.Default().PresignedGetObject("test", "test.apk", time.Second*24*60*60, reqParams)
if err != nil {
fmt.Println(err)
return
}
fmt.Println("Successfully generated presigned URL", presignedURL)
}
func main() {
//uploadTest()
//createBucketTest()
getTest()
}
go-oss/go-minio/uploader.go (3.2 KiB)
package gominio
import (
"context"
"errors"
golog "github.com/gif-gif/go.io/go-log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"log"
"net/url"
"time"
)
type Uploader struct {
conf Config
client *minio.Client
}
func Create(conf Config) (*Uploader, error) {
o := &Uploader{
conf: conf,
}
client, err := o.getClient()
if err != nil {
golog.Error(err.Error())
return nil, err
}
o.client = client
return o, nil
}
func (g *Uploader) GetConfig() Config {
return g.conf
}
func (g *Uploader) MinioClient() *minio.Client {
return g.client
}
func (o *Uploader) getClient() (*minio.Client, error) {
minioClient, err := minio.New(o.conf.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(o.conf.AccessKeyId, o.conf.AccessKeySecret, ""),
Secure: o.conf.UseSSL,
})
if err != nil {
log.Fatalln(err)
}
return minioClient, nil
}
func (o *Uploader) FPutObject(ctx context.Context, bucketName, objectName, filePath string, options *minio.PutObjectOptions) (*minio.UploadInfo, error) {
if objectName == "" {
return nil, errors.New("文件名为空")
}
if options == nil {
options = &minio.PutObjectOptions{ContentType: "application/octet-stream"}
}
info, err := o.client.FPutObject(ctx, bucketName, objectName, filePath, *options)
if err != nil {
return nil, err
}
return &info, nil
}
func (o *Uploader) GetObject(ctx context.Context, bucketName, objectName string, opts minio.GetObjectOptions) (*minio.Object, error) {
if objectName == "" {
return nil, errors.New("文件名为空")
}
return o.client.GetObject(ctx, bucketName, objectName, opts)
}
func (o *Uploader) FGetObject(ctx context.Context, bucketName, objectName, saveFilePath string, options *minio.GetObjectOptions) error {
if objectName == "" {
return errors.New("文件名为空")
}
if options == nil {
options = &minio.GetObjectOptions{}
}
err := o.client.FGetObject(ctx, bucketName, objectName, saveFilePath, *options)
if err != nil {
return err
}
return nil
}
// UploadExpiredObject
// UploadStreamObject
func (o *Uploader) CreateBucket(ctx context.Context, bucketName string, location string) error {
err := o.client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
if err != nil {
// Check to see if we already own this bucket (which happens if you run this twice)
exists, errBucketExists := o.client.BucketExists(ctx, bucketName)
if errBucketExists == nil && exists {
return nil
} else {
return err
}
}
return nil
}
func (o *Uploader) ListBuckets() ([]minio.BucketInfo, error) {
return o.client.ListBuckets(context.Background())
}
func (o *Uploader) BucketExists(bucketName string) (bool, error) {
return o.client.BucketExists(context.Background(), bucketName)
}
func (o *Uploader) ListObjects(bucketName string, opts *minio.ListObjectsOptions) <-chan minio.ObjectInfo {
if opts == nil {
opts = &minio.ListObjectsOptions{}
}
return o.client.ListObjects(context.Background(), bucketName, *opts)
}
func (o *Uploader) PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (u *url.URL, err error) {
return o.client.PresignedGetObject(context.Background(), bucketName, objectName, expiry, reqParams)
}
go-oss/readme.md (6 B)
OSS
go-pay/go-alipay/README.md (90 B)
go-zfb
支付宝登录、支付
官方SDK: https://github.com/smartwalle/alipay
go-pay/go-wechat/README.md (699 B)
go-wx
微信SDK,微信登录、支付
后续:实现微信菜单管理、H5授权、JSAPI、用户信息、小程序登录解析、小程序模板消息等
官方支付SDK
https://github.com/wechatpay-apiv3/wechatpay-go
使用Golang开发的微信SDK,简单、易用。doc: https://silenceper.com/wechat
https://github.com/silenceper/wechat
https://github.com/gowechat/example
微信机器人,利用微信号完成一些功能的定制化开发
https://github.com/eatmoreapple/openwechat
其他人封装库
- https://github.com/wizjin/weixin
- https://github.com/chanxuehong/wechat
- https://github.com/silenceper/wechat -- star 多
go-pay/go-wechat/test/main.go (2.9 KiB)
package main
import (
gocontext "github.com/gif-gif/go.io/go-context"
golog "github.com/gif-gif/go.io/go-log"
"github.com/silenceper/wechat/v2"
"github.com/silenceper/wechat/v2/cache"
"github.com/silenceper/wechat/v2/miniprogram/config"
offConfig "github.com/silenceper/wechat/v2/officialaccount/config"
"github.com/silenceper/wechat/v2/officialaccount/menu"
)
type AccessTokenHandle struct {
AccessToken string
}
func (a *AccessTokenHandle) GetAccessToken() (string, error) {
return a.AccessToken, nil
}
func main() {
testMiniProgram()
<-gocontext.WithCancel().Done()
}
func testMp() {
wc := wechat.NewWechat()
//这里本地内存保存access_token,也可选择redis,memcache或者自定cache
memory := cache.NewMemory()
cfg := &offConfig.Config{
AppID: "wxb19e7f16eafb98c2",
AppSecret: "b38828d87586ec284b093e0d87ca7b21",
Token: "123qwe",
//EncodingAESKey: "xxxx",
Cache: memory,
}
oa := wc.GetOfficialAccount(cfg)
//ak, err := oa.GetAccessToken()
//if err != nil {
// golog.WithTag("ak").Error(err)
// return
//}
//golog.WithTag("ak").Info(ak)
oa.SetAccessTokenHandle(&AccessTokenHandle{
AccessToken: "84_FO2Fgjm2MU73ylsnWriDGQ7XXLhNE792znTlmR_Cr7lnBeVK0ifnUcH-tzroqgUh2rW3tvSxWkfuIYH6TrN64Pl6B_V8PHswkCB-ZUrv6acFiC49p1C7avNO4dANMBaABADCR", //"84_sgRh9c8kz4umPXfg0qPveHGdj4E5kg6IUQUBTqrHK_GLnqnBtNDJxzHJLuIkgO-G6ttW2m-eMDmSmBIbgd0YvkAiQ1-UMIv9-O5dgjCPxlgLwnWdwHr60FK7jlITOKfAFABWS",
})
//ipList, err := officialAccount.GetBasic().GetCallbackIP()
ipList, err := oa.GetBasic().GetAPIDomainIP()
if err != nil {
golog.WithTag("ipList").Error(err)
return
}
golog.WithTag("ipList").Info(ipList)
bd := oa.GetBroadcast()
r, err := bd.SendText(nil, "hello")
if err != nil {
golog.WithTag("bd").Error(err)
return
}
golog.WithTag("bd").Info("ok", r)
m := oa.GetMenu()
var buttons []*menu.Button
buttons = append(buttons, &menu.Button{
Type: "click",
Name: "今日歌曲",
Key: "V1001_TODAY_MUSIC",
URL: "https://wx.acom.cc",
SubButtons: []*menu.Button{
&menu.Button{
Type: "click",
Name: "今日歌曲1",
Key: "V1001_TODAY_MUSIC1",
URL: "https://wx.acom.cc",
}, &menu.Button{
Type: "click",
Name: "今日歌曲2",
Key: "V1001_TODAY_MUSIC2",
URL: "https://wx.acom.cc",
},
},
})
err = m.SetMenu(buttons)
if err != nil {
golog.WithTag("m").Error(err)
return
}
u := oa.GetUser()
list, err := u.ListAllUserOpenIDs()
if err != nil {
golog.WithTag("list").Error(err)
return
}
golog.WithTag("list").Info(list)
golog.WithTag("m").Info("ok", r)
}
func testMiniProgram() {
wc := wechat.NewWechat()
memory := cache.NewMemory()
cfg := &config.Config{
AppID: "wxb19e7f16eafb98c2",
AppSecret: "b38828d87586ec284b093e0d87ca7b21",
Token: "123qwe",
//EncodingAESKey: "xxxx",
Cache: memory,
}
mini := wc.GetMiniProgram(cfg)
a := mini.GetAuth()
golog.WithTag("mini").Info(a)
}
go-pay/go-wechat/utils.go (640 B)
package gowechat
import (
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gogf/gf/util/gconv"
"math"
"sort"
"strings"
"time"
)
// 常用签名验证, sign Sha1 小写
func CheckSignSha1(secret, nonce string, linkSignTimeout int64, ts int64, sign string) bool {
if linkSignTimeout == 0 {
linkSignTimeout = 20
}
tsStep := time.Now().Unix() - ts
if math.Abs(gconv.Float64(tsStep)) > gconv.Float64(linkSignTimeout) { //连接超时
return false
}
args := []string{secret, gconv.String(ts), nonce}
sort.Strings(args)
// 小写
serverSign := goutils.SHA1([]byte(strings.Join(args, "")))
return serverSign == sign
}
go-pool/gopool.go (2.7 KiB)
package gopool
import (
"github.com/panjf2000/ants/v2"
"time"
)
type GoPool struct {
pool *ants.Pool
}
type PoolStat struct {
RunningWorkers int64
IdleWorkers int
SubmittedTasks uint64
WaitingTasks uint64
SuccessfulTasks uint64
FailedTasks uint64
CompletedTasks uint64
}
// ants will pre-malloc the whole capacity of pool when calling ants.NewPool.
//
// p, _ := ants.NewPool(100000, ants.WithPreAlloc(true))
func New(maxWorkers int, options ...ants.Option) (*GoPool, error) {
p, err := ants.NewPool(maxWorkers, options...)
return &GoPool{pool: p}, err
}
func (g *GoPool) GetPool() *ants.Pool {
return g.pool
}
func (g *GoPool) Submit(fn func()) error {
return g.pool.Submit(fn)
}
// 释放关闭此池并释放工作队列。
func (g *GoPool) Release() {
g.pool.Release()
}
// 只要调用 Reboot() 方法,就可以重新激活一个之前已经被销毁掉的池,并且投入使用。
func (g *GoPool) Reboot() {
g.pool.Reboot()
}
// ReleaseTimeout就像Release,但带有超时,等待所有工作者退出后再超时。
func (g *GoPool) ReleaseTimeout(timeout time.Duration) error {
return g.pool.ReleaseTimeout(timeout)
}
// 可用Worker数量,-1 表示无限制
func (g *GoPool) Free() {
g.pool.Free()
}
// 调整更改了此池的容量,请注意,这对无限池或预分配池是无效的。线程安全
func (g *GoPool) Tune(size int) {
g.pool.Tune(size)
}
func (g *GoPool) IsClosed() bool {
return g.pool.IsClosed()
}
// 运行中的Worker数量
func (g *GoPool) Running() int {
return g.pool.Running()
}
// 等待返回等待执行的任务数量。
func (g *GoPool) Waiting() int {
return g.pool.Waiting()
}
// Cap返回该池的容量
func (g *GoPool) Cap() int {
return g.pool.Cap()
}
//
//func (g *GoPool) PoolStats() PoolStat {
// return PoolStat{
// RunningWorkers: g.pool.RunningWorkers(),
// SubmittedTasks: g.pool.SubmittedTasks(),
// WaitingTasks: g.pool.WaitingTasks(),
// SuccessfulTasks: g.pool.SuccessfulTasks(),
// FailedTasks: g.pool.FailedTasks(),
// CompletedTasks: g.pool.CompletedTasks(),
// }
//}
//
//func (g *GoPool) PrintPoolStats() {
// ps := g.PoolStats()
// golog.WithTag("GoPool").InfoF("RunningWorkers: %d", ps.RunningWorkers)
// golog.WithTag("GoPool").InfoF("IdleWorkers: %d", ps.IdleWorkers)
// golog.WithTag("GoPool").InfoF("SubmittedTasks: %d", ps.SubmittedTasks)
// golog.WithTag("GoPool").InfoF("WaitingTasks: %d", ps.WaitingTasks)
// golog.WithTag("GoPool").InfoF("SuccessfulTasks: %d", ps.SuccessfulTasks)
// golog.WithTag("GoPool").InfoF("FailedTasks: %d", ps.FailedTasks)
// golog.WithTag("GoPool").InfoF("CompletedTasks: %d", ps.CompletedTasks)
// golog.WithTag("GoPool").InfoF("----------------------------------------------------------------")
//}
go-pool/readme.md (267 B)
GoPool for workers 并发请求 go routines 池
- 基于 https://github.com/panjf2000/ants 封装
Examples
pool, _ := gopool.New(100, ants.WithPreAlloc(true))
err := pool.Submit(func() {
})
if err != nil {
golog.ErrorF("Submit failed: %v", err)
}
go-pool/test/main.go (924 B)
package main
import (
"fmt"
golog "github.com/gif-gif/go.io/go-log"
gopool "github.com/gif-gif/go.io/go-pool"
"github.com/gif-gif/go.io/goio"
"github.com/panjf2000/ants/v2"
"time"
)
func timeCost(start time.Time) {
tc := time.Since(start)
fmt.Printf("time cost = %v\n", tc.Seconds())
}
// TODO: 并发怎么迅速取消或者停止,http 比如请求最快的直接返回
func main() {
goio.Init(goio.DEVELOPMENT)
defer timeCost(time.Now())
pool, _ := gopool.New(100, ants.WithPreAlloc(true))
err := pool.Submit(func() {
})
if err != nil {
golog.ErrorF("Submit failed: %v", err)
}
}
func testDynamicSize() {
golog.InfoF("end of Submit")
}
func testFixedSize() {
golog.InfoF("end of Submit")
}
func testContext() {
golog.InfoF("end of Submit")
}
func testTaskGroup() {
golog.InfoF("end of TaskGroup")
}
func testGroupContext() {
}
func testStopPool() {
golog.InfoF("Worker pool finished")
}
go-shell/tag.sh (1.4 KiB)
#!/bin/bash
# 脚本描述
echo "请选择要更新的版本号类型:"
echo "1. 更新主版本号(如 v1.0.0 -> v2.0.0)"
echo "2. 更新测试版本号(如 v0.1.0 -> v0.2.0)"
echo "3. 更新修订版本号(如 v0.0.1 -> v0.0.2)"
echo "默认使用修订版本号,直接按回车执行。"
# 读取用户输入
read -p "请输入选项(1/2/3,默认 3):" choice
# 设置默认值为 3(修订版本号)
choice=${choice:-3}
# 获取当前最新的 Tag
latest_tag=$(git describe --tags --abbrev=0 2>/dev/null)
echo "当前仓库最新tag版本号为:" $latest_tag
# 如果没有找到 Tag,默认从 v0.0.0 开始
if [ -z "$latest_tag" ]; then
latest_tag="v0.0.0"
fi
# 提取版本号(去掉开头的 'v')
version=${latest_tag#v}
# 将版本号拆分为主版本号、测试版本号和修订版本号
IFS='.' read -r major minor patch <<< "$version"
# 根据用户选择更新版本号
case $choice in
1)
major=$((major + 1))
minor=0
patch=0
;;
2)
minor=$((minor + 1))
patch=0
;;
3)
patch=$((patch + 1))
;;
*)
echo "无效选项,默认使用修订版本号。"
patch=$((patch + 1))
;;
esac
# 拼接新的版本号
new_tag="v${major}.${minor}.${patch}"
echo "本次即将提交的tag版本号为:" $new_tag
## 打 Tag
git tag "$new_tag"
git push origin "$new_tag"
# 输出最新的 Tag
echo "新的 Tag 已创建:$new_tag"
go-sso/go-jwt/client.go (991 B)
package gojwt
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoJwt{}
func Init(configs ...Config) error {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("gojwt client [" + name + "] already exists")
}
__clients[name] = New(conf)
}
return nil
}
func GetClient(names ...string) *GoJwt {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoJwt {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("gojwt").Error("no default jwt client")
return nil
}
go-sso/go-jwt/gojwt.go (3.1 KiB)
package gojwt
import (
"fmt"
"github.com/golang-jwt/jwt/v5"
"strings"
"time"
)
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
AccessSecret string `yaml:"AccessSecret" json:"accessSecret,optional"`
AccessExpire int64 `yaml:"AccessExpire" json:"accessExpire,optional"`
RefreshTokenExpire int64 `yaml:"RefreshTokenExpire" json:"refreshTokenExpire,optional"`
JwtEncryptSecret string `yaml:"JwtEncryptSecret" json:"jwtEncryptSecret,optional"` //Token加密秘钥
}
type GoJwt struct {
Config Config
}
func New(config Config) *GoJwt {
return &GoJwt{
Config: config,
}
}
func GetJwtToken(secretKey string, iat, seconds int64, params map[string]any) (string, error) {
claims := make(jwt.MapClaims)
claims["exp"] = iat + seconds
claims["iat"] = iat
for key, val := range params {
claims[key] = val
}
token := jwt.New(jwt.SigningMethodHS256)
token.Claims = claims
return token.SignedString([]byte(secretKey))
}
func (c *GoJwt) GeneratedTokens(params map[string]any) (accessToken, refreshToken string, expires int64, err error) {
now := time.Now()
expires = now.Add(time.Duration(c.Config.AccessExpire) * time.Second).Unix()
nowUnix := now.Unix()
accessToken, err = GetJwtToken(c.Config.AccessSecret, nowUnix, c.Config.AccessExpire, params)
if err != nil {
return
}
params["isRefreshToken"] = 1
refreshToken, err = GetJwtToken(c.Config.AccessSecret, nowUnix, c.Config.RefreshTokenExpire, params)
if err != nil {
return
}
return
}
// 解析token
func (c *GoJwt) ParseToken(tokenString string) (map[string]interface{}, error) {
if strings.Contains(tokenString, "Bearer ") {
tokenString = tokenString[7:]
}
token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
}
return []byte(c.Config.AccessSecret), nil
})
if err != nil {
return nil, err
}
claims, ok := token.Claims.(jwt.MapClaims)
if ok && token.Valid {
return claims, nil
} else {
return nil, err
}
}
func (c *GoJwt) IsValidToken(tokenString string) bool {
if strings.Contains(tokenString, "Bearer ") {
tokenString = strings.TrimSpace(tokenString[7:])
}
token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
}
return []byte(c.Config.AccessSecret), nil
})
if err != nil {
return false
}
return token.Valid
}
// 验证Refresh Token并生成新的Access Token
func (c *GoJwt) RefreshAccessToken(refreshTokenStr string) (accessToken, refreshToken string, expires int64, err error) {
if strings.Contains(refreshTokenStr, "Bearer ") {
refreshTokenStr = refreshTokenStr[7:]
}
if !c.IsValidToken(refreshTokenStr) {
return "", "", 0, fmt.Errorf("invalid refresh token")
}
params, err := c.ParseToken(refreshTokenStr)
if err != nil {
return "", "", 0, fmt.Errorf("invalid refresh token")
}
return c.GeneratedTokens(params)
}
go-sso/go-jwt/test/main.go (764 B)
package main
import (
golog "github.com/gif-gif/go.io/go-log"
gojwt "github.com/gif-gif/go.io/go-sso/go-jwt"
)
func main() {
c := gojwt.Config{
AccessSecret: "111",
AccessExpire: 86400 * 360 * 10,
RefreshTokenExpire: 86400 * 360 * 10,
}
params := map[string]any{
"aaa": 1,
}
gojwt.Init(c)
a, r, e, err := gojwt.Default().GeneratedTokens(params)
if err != nil {
golog.Error(err.Error())
return
}
golog.Info(a, r, e)
m, err := gojwt.Default().ParseToken(a)
if err != nil {
golog.Error(err.Error())
return
}
golog.Info(m)
if gojwt.Default().IsValidToken(a) {
golog.WithTag("IsValidToken").Info("OK")
}
a, b, cc, err := gojwt.Default().RefreshAccessToken(r)
golog.WithTag("RefreshToken").Info(a, b, cc, err)
}
go-sso/go-jwt/test1/test.go (357 B)
package main
import "flag"
var gopath string
func init() {
println("init a")
}
func init() {
println("init b")
}
func init() {
println("init c")
// gopath may be overridden by --gopath flag on command line.
flag.StringVar(&gopath, "gopath", "/root/go", "override default GOPATH")
}
func main() {
println("main")
flag.Parse()
println(gopath)
}
go-sso/go-oauth/client.go (1.1 KiB)
package gooauth
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoOAuth{}
// 可以一次初始化多个实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("GoOAuth [" + name + "] already exists")
}
__clients[name] = New(conf)
}
return
}
func GetClient(names ...string) *GoOAuth {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoOAuth {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("GoOAuth").Error("no default GoOAuth client")
return nil
}
go-sso/go-oauth/go-oauth.go (3.7 KiB)
package gooauth
import (
"context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
)
type Endpoint struct {
TokenURL string `yaml:"TokenURL" json:"tokenURL,optional"`
AuthURL string `yaml:"AuthURL" json:"authURL,optional"`
DeviceAuthURL string `yaml:"DeviceAuthURL" json:"deviceAuthURL,optional"`
AuthStyle oauth2.AuthStyle `yaml:"AuthStyle" json:"authStyle,optional"`
} // `yaml:"Endpoint" json:"endpoint"`
type Config struct {
Name string `yaml:"Name" json:"name,optional"`
// Token 信息
AccessToken string `yaml:"AccessToken" json:"accessToken,optional"`
RefreshToken string `yaml:"RefreshToken" json:"refreshToken,optional"`
ExpiresIn int64 `yaml:"ExpiresIn" json:"expiresIn,optional"` //秒s
// 授权参数 OAuthConfig oauth2.Config
ClientId string `yaml:"ClientId" json:"clientId,optional"`
ClientSecret string `yaml:"ClientSecret" json:"clientSecret,optional"`
RedirectURL string `yaml:"RedirectURL" json:"redirectURL,optional"`
Endpoint *Endpoint `yaml:"Endpoint" json:"endpoint"`
Scopes []string `yaml:"Scopes" json:"scopes,optional"`
}
type GoOAuth struct {
Config Config
Token *oauth2.Token
OAuthConfig oauth2.Config
}
func New(config Config) *GoOAuth {
if config.Endpoint == nil { //默认google 平台
config.Endpoint = &Endpoint{
TokenURL: google.Endpoint.TokenURL,
AuthURL: google.Endpoint.AuthURL,
DeviceAuthURL: google.Endpoint.DeviceAuthURL,
AuthStyle: google.Endpoint.AuthStyle,
}
}
oConfig := oauth2.Config{
ClientID: config.ClientId,
ClientSecret: config.ClientSecret,
RedirectURL: config.RedirectURL,
Endpoint: oauth2.Endpoint{
TokenURL: config.Endpoint.TokenURL,
AuthURL: config.Endpoint.AuthURL,
DeviceAuthURL: config.Endpoint.DeviceAuthURL,
AuthStyle: config.Endpoint.AuthStyle,
},
Scopes: config.Scopes,
}
token := &oauth2.Token{ //兼容处理,初始化的token
AccessToken: config.AccessToken,
RefreshToken: config.RefreshToken,
ExpiresIn: config.ExpiresIn,
}
return &GoOAuth{
Token: token,
OAuthConfig: oConfig,
Config: config,
}
}
func (c *GoOAuth) TokenSource(ctx context.Context) oauth2.TokenSource {
return c.OAuthConfig.TokenSource(ctx, c.Token)
}
// 获取授权url
//
// 自定义参数
//
// param1 := oauth2.SetAuthURLParam("param1", "value1")
// param2 := oauth2.SetAuthURLParam("param2", "value2")
func (c *GoOAuth) AuthUrl(state string, opts ...oauth2.AuthCodeOption) string {
url := c.OAuthConfig.AuthCodeURL(state, opts...)
return url
}
// 获取token
//
// 自定义参数
//
// param1 := oauth2.SetAuthURLParam("param1", "value1")
// param2 := oauth2.SetAuthURLParam("param2", "value2")
func (c *GoOAuth) Exchange(ctx context.Context, authorizationCode string, opts ...oauth2.AuthCodeOption) (*oauth2.Token, error) {
token, err := c.OAuthConfig.Exchange(ctx, authorizationCode, opts...)
if err != nil {
return nil, err
}
c.Config.AccessToken = token.AccessToken
c.Config.RefreshToken = token.RefreshToken
c.Token = token
return token, nil
}
// 刷新token
func (c *GoOAuth) RefreshToken(ctx context.Context) (*oauth2.Token, error) {
token, err := c.OAuthConfig.TokenSource(ctx, &oauth2.Token{
RefreshToken: c.Config.RefreshToken,
}).Token()
if err != nil {
return nil, err
}
c.Config.AccessToken = token.AccessToken
c.Config.RefreshToken = token.RefreshToken
c.Token = token
return token, nil
}
// 详细的信息,注意:只有在刷新过token的时候才会有值
func (c *GoOAuth) GetToken() *oauth2.Token {
return c.Token
}
func (c *GoOAuth) GetAccessToken() string {
return c.Config.AccessToken
}
func (c *GoOAuth) GetRefreshToken() string {
return c.Config.RefreshToken
}
go-sso/go-oauth/go-oauth_test.go (834 B)
package gooauth
import (
golog "github.com/gif-gif/go.io/go-log"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"testing"
)
func TestAdmobAuthUrl(t *testing.T) {
Init(Config{
Name: "test",
ClientId: "123",
ClientSecret: "123",
RedirectURL: "URL_ADDRESS",
Scopes: []string{"URL_ADDRESS"},
Endpoint: &Endpoint{
AuthURL: google.Endpoint.AuthURL,
TokenURL: google.Endpoint.TokenURL,
//AuthStyle: google.Endpoint.AuthStyle,
AuthStyle: oauth2.AuthStyleInHeader,
},
})
param2 := oauth2.SetAuthURLParam("param2", "value2")
url := GetClient("test").AuthUrl("ppp=====aa", param2)
golog.WithTag("admob").Info(url)
//golog.WithTag("admob").Info(goutils.UrlDecode("https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fadmob.readonly+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fadmob.report"))
}
go-sso/go-oauth/readme.md (81 B)
Go OAuth2.0
基于 golang.org/x/oauth2
封装 通用OAuth2.0常用方法
go-sso/go-oauth/types.go (961 B)
package gooauth
import (
"context"
"golang.org/x/oauth2"
)
// GoogleUserInfo 存储从 Google API 获取的用户信息
type GoogleUserInfo struct {
ID string `json:"id"`
Email string `json:"email"`
VerifiedEmail bool `json:"verified_email"`
Name string `json:"name"`
GivenName string `json:"given_name"`
FamilyName string `json:"family_name"`
Picture string `json:"picture"`
Locale string `json:"locale"`
}
// OAuth2.0 授权接口, 支持多种授权方式
type IGoOAuth interface {
// 自定义参数
//
// param1 := oauth2.SetAuthURLParam("param1", "value1")
// param2 := oauth2.SetAuthURLParam("param2", "value2")
AuthUrl(opts ...oauth2.AuthCodeOption) string
Exchange(ctx context.Context, authorizationCode string, opts ...oauth2.AuthCodeOption) (*oauth2.Token, error)
RefreshToken(ctx context.Context, opts ...oauth2.AuthCodeOption) (*oauth2.Token, error)
GetToken() *oauth2.Token
}
go-sso/go-oauth/utils.go (1.8 KiB)
package gooauth
import (
"context"
"encoding/json"
"fmt"
"google.golang.org/api/idtoken"
"google.golang.org/api/option"
"io"
"net/http"
)
func VerifyGoogleToken(accessToken string, clientId string, ctx context.Context) (*GoogleUserInfo, error) {
// 创建验证器
validator, err := idtoken.NewValidator(ctx, option.WithoutAuthentication())
if err != nil {
return nil, fmt.Errorf("failed to create validator: %v", err)
}
// 验证 ID Token
payload, err := validator.Validate(ctx, accessToken, clientId)
if err != nil {
return nil, fmt.Errorf("invalid token: %v", err)
}
// 从 payload 创建 GoogleUser 结构体
googleUser := GoogleUserInfo{
Email: payload.Claims["email"].(string),
VerifiedEmail: payload.Claims["email_verified"].(bool),
Name: payload.Claims["name"].(string),
GivenName: payload.Claims["given_name"].(string),
FamilyName: payload.Claims["family_name"].(string),
Picture: payload.Claims["picture"].(string),
Locale: "",
}
return &googleUser, nil
}
func GetUserInfoFromGoogle(accessToken string) (*GoogleUserInfo, error) {
// 创建HTTP请求获取用户信息
resp, err := http.Get("https://www.googleapis.com/oauth2/v2/userinfo?access_token=" + accessToken)
if err != nil {
return nil, fmt.Errorf("获取用户信息失败: %v", err)
}
defer resp.Body.Close()
// 读取响应
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("读取响应失败: %v", err)
}
// 检查HTTP状态码
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("API响应错误: %s", string(data))
}
// 解析用户信息
var userInfo GoogleUserInfo
if err := json.Unmarshal(data, &userInfo); err != nil {
return nil, fmt.Errorf("解析用户信息失败: %v", err)
}
return &userInfo, nil
}
go-test/readme.md (156 B)
Test Dir
测试程序目录 Testing for anything dir
没有具体的模块,纯粹测试区域目录
这个目录下的测试代码不要提交
go-tgbot/readme.md (2.8 KiB)
Golang 控制tg相关api封装
- 基于 https://github.com/tucnak/telebot
package main
import (
golog "github.com/gif-gif/go.io/go-log"
gotgbot "github.com/gif-gif/go.io/go-tgbot"
"github.com/gogf/gf/util/gconv"
tele "gopkg.in/telebot.v3"
"time"
)
// Recipient returns chat ID (see Recipient interface).
func main() {
var chatId int64 = 5562314141
pref := &gotgbot.TelegramBot{
Product: "test",
Token: "Token",
WebAppUrl: "https://www.google.com",
StartReply: "hello",
}
gobot, err := gotgbot.CreateOfflineBot(pref) // TODO离线模式测试
if err != nil {
golog.Error(err.Error())
return
}
var (
// Universal markup builders.
menu = &tele.ReplyMarkup{
InlineKeyboard: [][]tele.InlineButton{{
{
Text: "This is a web app",
WebApp: &tele.WebApp{URL: "https://google.com"},
},
}},
}
)
gobot.Handle("/start", func(c tele.Context) error {
return c.Send(pref.StartReply, menu)
})
var trace []string
handler := func(name string) tele.HandlerFunc {
return func(c tele.Context) error {
trace = append(trace, name)
return nil
}
}
middleware := func(c tele.Context, next tele.HandlerFunc, params ...any) error {
trace = append(trace, gconv.String(params[0])+":in")
err := next(c)
trace = append(trace, gconv.String(params[0])+":out")
return err
}
createMiddleware1 := gobot.CreateMiddleware(middleware, "handler1a")
createMiddleware2 := gobot.CreateMiddleware(middleware, "handler2a")
createGroup1Middleware1 := gobot.CreateMiddleware(middleware, "group1")
createGroup2Middleware2 := gobot.CreateMiddleware(middleware, "group2")
createGroup2Middleware3 := gobot.CreateMiddleware(middleware, "handler1b")
b := gobot.GetBot()
gobot.UseMiddleware(middleware, "test")
gobot.UseMiddleware(middleware, "global1")
gobot.UseMiddleware(middleware, "global2")
b.Handle("/a", handler("/a"), createMiddleware1, createMiddleware2)
group := b.Group()
group.Use(createGroup1Middleware1, createGroup2Middleware2)
group.Handle("/b", handler("/b"), createGroup2Middleware3)
b.ProcessUpdate(tele.Update{
Message: &tele.Message{Text: "/a"},
})
golog.WithTag("gotgbot").Info(trace)
trace = trace[:0]
b.ProcessUpdate(tele.Update{
Message: &tele.Message{Text: "/b"},
})
golog.WithTag("gotgbot").Info(trace)
gobot.SendMsgText(chatId, "text")
gobot.SendFromUrlPhotos(gotgbot.ChatID(chatId), []string{"https://developer.android.google.cn/static/studio/images/run/adb_wifi-quick_settings.png?hl=zh-cn"})
go gobot.StartBot()
time.Sleep(1000 * time.Second)
}
旧
- https://github.com/go-telegram-bot-api/telegram-bot-api
- https://github.com/m1guelpf/chatgpt-telegram
bot 仓库
- https://github.com/tucnak/telebot
- https://github.com/PaulSonOfLars/gotgbot
- https://github.com/go-telegram/bot
- https://github.com/EverythingSuckz/TG-FileStreamBot
go-tgbot/test/main.go (2.3 KiB)
package main
import (
golog "github.com/gif-gif/go.io/go-log"
gotgbot "github.com/gif-gif/go.io/go-tgbot"
"time"
)
// Recipient returns chat ID (see Recipient interface).
func main() {
var chatId int64 = 5562314141
pref := &gotgbot.TelegramBot{
Product: "test",
Token: "",
WebAppUrl: "https://www.google.com",
StartReply: "hello",
ApiUrl: "https://api.jidianle.cc/tg",
}
gobot, err := gotgbot.Create(pref) // TODO离线模式测试
if err != nil {
golog.Error(err.Error())
return
}
gobot.CreateMyAccountCommand("/account")
var trace []string
//handler := func(name string) tele.HandlerFunc {
// return func(c tele.Context) error {
// gotrace = append(gotrace, name)
// return nil
// }
//}
//
//middleware := func(c tele.Context, next tele.HandlerFunc, params ...any) error {
// gotrace = append(gotrace, gconv.String(params[0])+":in")
// err := next(c)
// gotrace = append(gotrace, gconv.String(params[0])+":out")
// return err
//}
//
//createMiddleware1 := gobot.CreateMiddleware(middleware, "handler1a")
//createMiddleware2 := gobot.CreateMiddleware(middleware, "handler2a")
//createGroup1Middleware1 := gobot.CreateMiddleware(middleware, "group1")
//createGroup2Middleware2 := gobot.CreateMiddleware(middleware, "group2")
//createGroup2Middleware3 := gobot.CreateMiddleware(middleware, "handler1b")
//b := gobot.GetBot()
//gobot.UseMiddleware(middleware, "test")
//gobot.UseMiddleware(middleware, "global1")
//gobot.UseMiddleware(middleware, "global2")
//b.Handle("/a", handler("/a"), createMiddleware1, createMiddleware2)
//
//group := b.Group()
//group.Use(createGroup1Middleware1, createGroup2Middleware2)
//group.Handle("/b", handler("/b"), createGroup2Middleware3)
//b.ProcessUpdate(tele.Update{
// Message: &tele.Message{Text: "/a"},
//})
golog.WithTag("gotgbot").Info(trace)
trace = trace[:0]
//b.ProcessUpdate(tele.Update{
// Message: &tele.Message{Text: "/b"},
//})
golog.WithTag("gotgbot").Info(trace)
gobot.SendMsgText(chatId, "text")
//gobot.SendFromUrlPhotos(gotgbot.ChatID(chatId), []string{
// "https://developer.android.google.cn/static/studio/images/run/adb_wifi-quick_settings.png?hl=zh-cn",
// "https://biuvip.com/bot/how_agents_buy_tokens.png",
//})
go gobot.StartBot()
//gobot.StopBot()
time.Sleep(1000 * time.Second)
}
go-tgbot/tgbot.go (5.9 KiB)
package gotgbot
import (
"errors"
"fmt"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gogf/gf/crypto/gmd5"
"github.com/gogf/gf/util/gconv"
"gopkg.in/telebot.v3"
"time"
)
type GoTgBot struct {
bot *telebot.Bot
reply *telebot.ReplyMarkup
config *TelegramBot
started bool
}
// 离线测试, 模拟用户发命令
//
// b.ProcessUpdate(tele.Update{
// Message: &tele.Message{Text: "/a"},
// })
func CreateOfflineBot(config *TelegramBot) (*GoTgBot, error) {
bot, err := telebot.NewBot(telebot.Settings{Synchronous: true, Offline: true})
if err != nil {
return nil, err
}
return &GoTgBot{
bot: bot,
reply: bot.NewMarkup(),
}, nil
}
// 同一个产品只会存在一个
func Create(config *TelegramBot) (*GoTgBot, error) {
if config.Product == "" {
return nil, fmt.Errorf("product must be empty")
}
if config.Timeout == 0 {
config.Timeout = 10
}
if config.SignTimeout == 0 {
config.SignTimeout = 20
}
pref := telebot.Settings{
Token: config.Token,
URL: config.ApiUrl,
Offline: config.Offline,
Synchronous: config.Synchronous,
ParseMode: config.ParseMode,
Poller: &telebot.LongPoller{Timeout: time.Duration(config.Timeout) * time.Second},
}
bot, err := telebot.NewBot(pref)
if err != nil {
return nil, err
}
return &GoTgBot{
bot: bot,
config: config,
reply: bot.NewMarkup(),
}, nil
}
// /start
func (g *GoTgBot) CreateWebAppStartCommand(menuButtonText string) {
var (
menu = &telebot.ReplyMarkup{
InlineKeyboard: [][]telebot.InlineButton{{
{
Text: menuButtonText,
WebApp: &telebot.WebApp{URL: g.config.WebAppUrl},
},
}},
}
)
g.Handle("/start", func(c telebot.Context) error {
return c.Send(g.config.StartReply, menu)
})
}
func (g *GoTgBot) CreateMyAccountCommand(commandText string) {
g.Handle(commandText, func(c telebot.Context) error {
accountMd5, err := gmd5.Encrypt(gconv.String(c.Chat().ID))
if err != nil {
return err
}
_, err = g.SendMsgText(c.Chat().ID, accountMd5)
if err != nil {
return errors.New("failed to send message")
}
return nil
})
}
func (g *GoTgBot) GetBot() *telebot.Bot {
return g.bot
}
func (g *GoTgBot) StopBot() {
if !g.started {
return
}
g.started = false
g.bot.Stop()
}
func (g *GoTgBot) StartBot() {
g.started = true
g.bot.Start()
}
func (g *GoTgBot) GetConfig() *TelegramBot {
return g.config
}
func (g *GoTgBot) SendMsgText(chatId int64, msg string, opts ...interface{}) (*telebot.Message, error) {
return g.Send(ChatID(chatId), msg, opts...)
}
func (g *GoTgBot) Send(to telebot.Recipient, what interface{}, opts ...interface{}) (*telebot.Message, error) {
return g.bot.Send(to, what, opts...)
}
func (g *GoTgBot) SendAlbum(to telebot.Recipient, a telebot.Album, opts ...interface{}) ([]telebot.Message, error) {
return g.bot.SendAlbum(to, a, opts...)
}
func (g *GoTgBot) BanSenderChat(chat *telebot.Chat, sender telebot.Recipient) error {
return g.bot.BanSenderChat(chat, sender)
}
func (g *GoTgBot) Handle(endpoint interface{}, h telebot.HandlerFunc, m ...telebot.MiddlewareFunc) {
g.bot.Handle(endpoint, h, m...)
}
func (g *GoTgBot) SendFromDiskPhotos(to telebot.Recipient, fromDiskFiles []string) ([]telebot.Message, error) {
albums := telebot.Album{}
for _, file := range fromDiskFiles {
albums = append(albums, &telebot.Photo{File: telebot.FromDisk(file)})
}
return g.SendAlbum(to, albums)
}
func (g *GoTgBot) SendFromUrlPhotos(to telebot.Recipient, urls []string) ([]telebot.Message, error) {
albums := telebot.Album{}
for _, url := range urls {
albums = append(albums, &telebot.Photo{File: telebot.FromURL(url)})
}
return g.SendAlbum(to, albums)
}
func (g *GoTgBot) SendFromDiskVideos(to telebot.Recipient, fromDiskFiles []string) ([]telebot.Message, error) {
//v := &telebot.Video{File: telebot.FromURL("http://video.mp4")}
albums := telebot.Album{}
for _, file := range fromDiskFiles {
albums = append(albums, &telebot.Video{File: telebot.FromDisk(file)})
}
return g.SendAlbum(to, albums)
}
func (g *GoTgBot) SendFromUrlVideos(to telebot.Recipient, urls []string) ([]telebot.Message, error) {
//v := &telebot.Video{File: telebot.FromURL("http://video.mp4")}
albums := telebot.Album{}
for _, url := range urls {
albums = append(albums, &telebot.Video{File: telebot.FromURL(url)})
}
return g.SendAlbum(to, albums)
}
func (g *GoTgBot) SendFromUrlAudios(to telebot.Recipient, urls []string) ([]telebot.Message, error) {
rest := []telebot.Message{}
for _, url := range urls {
r, err := g.Send(to, &telebot.Audio{File: telebot.FromURL(url)})
if err != nil {
return nil, err
}
rest = append(rest, *r)
}
return rest, nil
}
func (g *GoTgBot) SendFromDiskAudios(to telebot.Recipient, fromDiskFiles []string) ([]telebot.Message, error) {
rest := []telebot.Message{}
for _, file := range fromDiskFiles {
r, err := g.Send(to, &telebot.Audio{File: telebot.FromDisk(file)})
if err != nil {
return nil, err
}
rest = append(rest, *r)
}
return rest, nil
}
func (g *GoTgBot) Use(middleware ...telebot.MiddlewareFunc) {
g.bot.Use(middleware...)
}
func (g *GoTgBot) UseMiddleware(middleware func(c telebot.Context, next telebot.HandlerFunc, parameters ...any) error, parameters ...any) {
mm := func(next telebot.HandlerFunc) telebot.HandlerFunc {
return func(c telebot.Context) error {
err := middleware(c, next, parameters...)
return err
}
}
g.bot.Use(mm)
}
func (g *GoTgBot) CreateMiddleware(middleware func(c telebot.Context, next telebot.HandlerFunc, parameters ...any) error, parameters ...any) telebot.MiddlewareFunc {
mm := func(next telebot.HandlerFunc) telebot.HandlerFunc {
return func(c telebot.Context) error {
err := middleware(c, next, parameters...)
return err
}
}
return mm
}
func (g *GoTgBot) ReplyMarkup() *telebot.ReplyMarkup {
return g.reply
}
func (g *GoTgBot) CheckSign(ts int64, sign string) bool {
return goutils.CheckSign(g.config.SignSecret, g.config.SignTimeout, ts, sign)
}
go-tgbot/tgbots.go (1015 B)
package gotgbot
import (
"fmt"
)
type GoTgBots struct {
bots map[string]*GoTgBot
}
func New() *GoTgBots {
return &GoTgBots{
bots: make(map[string]*GoTgBot),
}
}
// 同一个产品只会存在一个,后一个添加的覆盖前面添加
func (g *GoTgBots) CreateBot(config *TelegramBot) (*GoTgBot, error) {
if config.Product == "" {
return nil, fmt.Errorf("product must be empty")
}
old := g.bots[config.Product]
if old != nil {
return nil, fmt.Errorf("Already exists a bot for %s", config.Product)
}
bot, err := Create(config)
if err != nil {
return nil, err
}
g.bots[config.Product] = bot
return bot, nil
}
func (g *GoTgBots) StopBot(product string) {
bot := g.GetBot(product)
bot.StopBot()
}
func (g *GoTgBots) StartBot(product string) {
bot := g.GetBot(product)
bot.StartBot()
}
func (g *GoTgBots) DestroyBot(product string) {
bot := g.GetBot(product)
bot.StopBot()
delete(g.bots, product)
}
func (g *GoTgBots) GetBot(product string) *GoTgBot {
return g.bots[product]
}
go-tgbot/tgext/tgext.go (3.1 KiB)
package tgext
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/url"
"sort"
"strings"
)
// ValidateLoginQuery validates a login widget query.
// See https://core.telegram.org/widgets/login#checking-authorization for more details.
func ValidateLoginQuery(query url.Values, token string) (bool, error) {
tokenHash, err := getSHA256(token)
if err != nil {
return false, fmt.Errorf("failed to hash token: %w", err)
}
return validateQuery(query, tokenHash)
}
// ValidateWebAppInitData validates a webapp's initData field for safe use on the server-side.
// The initData field is stored as a query string, so this is converted and then validated.
// See https://core.telegram.org/bots/webapps#validating-data-received-via-the-web-app for more details.
func ValidateWebAppInitData(initData string, token string) (bool, error) {
query, err := url.ParseQuery(initData)
if err != nil {
return false, fmt.Errorf("failed to parse URL query: %w", err)
}
return ValidateWebAppQuery(query, token)
}
// ValidateWebAppQuery validates a webapp's initData query for safe use on the server side.
// The input is expected to be the parsed initData query string.
// See https://core.telegram.org/bots/webapps#validating-data-received-via-the-web-app for more details.
func ValidateWebAppQuery(query url.Values, token string) (bool, error) {
tokenHMAC, err := generateHMAC256(token, []byte("WebAppData"))
if err != nil {
return false, fmt.Errorf("failed to generate token HMAC: %w", err)
}
return validateQuery(query, tokenHMAC)
}
func validateQuery(query url.Values, secretKey []byte) (bool, error) {
// If no hash, we can't check; fail-fast.
hash := query.Get("hash")
if hash == "" {
// Should this be an error?
return false, nil
}
// Make list of args for ordered sorting.
// len()-1, because we ignore the hash key.
args := make([]string, 0, len(query)-1)
for x, y := range query {
if x == "hash" {
// ignore the hash
continue
}
args = append(args, x+"="+y[0])
}
// Sort args to ensure consistency.
sort.Strings(args)
// Join data with newline, as defined by telegram.
dataCheck := strings.Join(args, "\n")
// Generate HMAC of expected data.
expectedHMAC, err := generateHMAC256(dataCheck, secretKey)
if err != nil {
return false, fmt.Errorf("failed to generate data HMAC: %w", err)
}
// Hex encode expected hmac_256 value.
expectedHex := getHex(expectedHMAC)
// Check hash matches as expected.
return hmac.Equal(expectedHex, []byte(hash)), nil
}
func generateHMAC256(data string, secretKey []byte) ([]byte, error) {
hmac256Writer := hmac.New(sha256.New, secretKey)
_, err := hmac256Writer.Write([]byte(data))
if err != nil {
return nil, fmt.Errorf("failed to write HMAC256: %w", err)
}
return hmac256Writer.Sum(nil), nil
}
func getSHA256(data string) ([]byte, error) {
sha256Writer := sha256.New()
_, err := sha256Writer.Write([]byte(data))
if err != nil {
return nil, fmt.Errorf("failed to write SHA256: %w", err)
}
return sha256Writer.Sum(nil), nil
}
func getHex(expectedHMAC []byte) []byte {
expectedHex := make([]byte, hex.EncodedLen(len(expectedHMAC)))
hex.Encode(expectedHex, expectedHMAC)
return expectedHex
}
go-tgbot/types.go (1.4 KiB)
package gotgbot
import (
"gopkg.in/telebot.v3"
"strconv"
)
type TelegramBot struct {
Product string `yaml:"Product" json:"product"`
ApiUrl string `yaml:"ApiUrl" json:"apiUrl,optional"`
ParseMode telebot.ParseMode `yaml:"ParseMode" json:"parseMode,optional"`
Token string `yaml:"Token" json:"token"`
Timeout int64 `yaml:"Timeout" json:"timeout,optional"` //s tg bot api timeout
Offline bool `yaml:"Offline" json:"offline,optional"`
Synchronous bool `yaml:"Synchronous" json:"synchronous,optional"`
// Product configuration (可选)
Open bool `yaml:"Open" json:"open,optional"` //是否开启
WebAppUrl string `yaml:"WebAppUrl" json:"webAppUrl,optional"`
StartReply string `yaml:"StartReply" json:"startReply,optional"`
SignSecret string `yaml:"SignSecret" json:"signSecret,optional"` //请求签名密钥
SignTimeout int64 `yaml:"SignTimeout" json:"signTimeout,optional"` //请求签名过期时间-秒
AccessSecret string `yaml:"AccessSecret" json:"accessSecret,optional"` //jwt signature secret
AccessExpire int64 `yaml:"AccessExpire" json:"accessExpire,optional"` // jwt signature accessExpire
}
type ChatID int64
// Recipient returns chat ID (see Recipient interface).
func (i ChatID) Recipient() string {
return strconv.FormatInt(int64(i), 10)
}
go-utils/array.go (2.6 KiB)
package goutils
import "slices"
func SplitStringArray(arr []string, size int) (list [][]string) {
l := len(arr)
if l == 0 {
list = make([][]string, 0)
return
}
if l < size {
list = [][]string{arr}
return
}
var (
offset int
)
for {
if offset+size >= l {
list = append(list, arr[offset:])
break
}
list = append(list, arr[offset:offset+size])
offset += size
}
return
}
func SplitIntArray(arr []int, size int) (list [][]int) {
l := len(arr)
if l == 0 {
list = make([][]int, 0)
return
}
if l < size {
list = [][]int{arr}
return
}
var (
offset int
)
for {
if offset+size >= l {
list = append(list, arr[offset:])
break
}
list = append(list, arr[offset:offset+size])
offset += size
}
return
}
func SplitInt64Array(arr []int64, size int) (list [][]int64) {
l := len(arr)
if l == 0 {
list = make([][]int64, 0)
return
}
if l < size {
list = [][]int64{arr}
return
}
var (
offset int
)
for {
if offset+size >= l {
list = append(list, arr[offset:])
break
}
list = append(list, arr[offset:offset+size])
offset += size
}
return
}
func SplitArray(arr []interface{}, size int) (list [][]interface{}) {
l := len(arr)
if l == 0 {
list = make([][]interface{}, 0)
return
}
if l < size {
list = [][]interface{}{arr}
return
}
var (
offset int
)
for {
if offset+size >= l {
list = append(list, arr[offset:])
break
}
list = append(list, arr[offset:offset+size])
offset += size
}
return
}
//
//// Insert 在指定位置插入元素
//// 支持负数索引,-1 表示最后一个位置,-2 表示倒数第二个位置
//func InsertArray[T any](slice []T, index int, element T) []T {
// // 处理负数索引
// if index < 0 {
// index = len(slice) + index + 1
// }
//
// // 边界检查
// if index < 0 {
// index = 0
// } else if index > len(slice) {
// index = len(slice)
// }
//
// // 插入元素
// return append(slice[:index], append([]T{element}, slice[index:]...)...)
//}
// Comparator 函数类型,接受两个参数并返回整数:
// - 负数表示 a < b
// - 零表示 a == b
// - 正数表示 a > b
type Comparator[T any] func(a, b T) int
// SortWith 使用多个比较函数对切片进行排序
func SortWith[T any](fns []Comparator[T], list []T) []T {
result := make([]T, len(list))
copy(result, list)
// 使用自定义排序逻辑
slices.SortStableFunc(result, func(a, b T) int {
for _, fn := range fns {
if cmp := fn(a, b); cmp != 0 {
return cmp
}
}
return 0 // 所有比较函数都返回 0,保持原顺序
})
return result
}
go-utils/async.go (2.4 KiB)
package goutils
import (
"context"
golog "github.com/gif-gif/go.io/go-log"
"github.com/zeromicro/go-zero/core/logx"
"golang.org/x/sync/errgroup"
"sync"
"time"
)
type ErrorGroup struct {
maxWorkers int
errGroup *errgroup.Group
ctx context.Context
}
// 当并发执行过程中有错误时,会自动取消其他所有任务,通过CancelContext 取消来实现
// (最大并发数为 maxWorkers,超过阻塞等待 )
func NewErrorGroup(context context.Context, maxWorkers int) ErrorGroup {
e := ErrorGroup{}
e.maxWorkers = maxWorkers
e.errGroup, e.ctx = errgroup.WithContext(context)
e.errGroup.SetLimit(e.maxWorkers)
return e
}
func (e *ErrorGroup) GetContext() context.Context {
return e.ctx
}
func (e *ErrorGroup) Submit(fn ...func() error) {
for _, f := range fn {
e.errGroup.Go(f)
}
}
func (e *ErrorGroup) TryGo(fn func() error) bool {
return e.errGroup.TryGo(fn)
}
func (e *ErrorGroup) Wait() error {
return e.errGroup.Wait()
}
func (e *ErrorGroup) IsContextDone() bool {
return IsContextDone(e.ctx)
}
// 捕获panic
func Recovery(errFn func(err any)) {
if r := recover(); r != nil {
if errFn != nil {
errFn(r)
} else {
logx.Errorf("AsyncFunc recover: %v", r)
golog.Error(r)
}
}
}
// 异步执行(安全)errFn = nil 时自动Recovery 不会Panic
func AsyncFuncPanic(fn func(), errFn func(err any)) {
go func() {
defer Recovery(errFn)
fn()
}()
}
// 异步执行(安全) RunSafe
func AsyncFunc(fn func()) {
go func() {
defer Recovery(nil)
fn()
}()
}
// 异步并发执行(安全), 建议使用NewErrorGroup 替代
func AsyncFuncGroup(fns ...func()) {
var wg sync.WaitGroup
for _, fn := range fns {
wg.Add(1)
func(fn func()) {
AsyncFunc(func() {
defer wg.Done()
fn()
})
}(fn)
}
wg.Wait()
}
// 异步并发执行(安全)errFn = nil 时自动Recovery 不会Panic
//
// 建议使用NewErrorGroup 替代
func AsyncFuncGroupPanic(errFn func(err any), fns ...func()) {
var wg sync.WaitGroup
for _, fn := range fns {
wg.Add(1)
func(fn func()) {
AsyncFuncPanic(func() {
defer wg.Done()
fn()
}, errFn)
}(fn)
}
wg.Wait()
}
// 返回函数执行时间
func MeasureExecutionTime(fn func()) time.Duration {
start := time.Now()
fn()
return time.Since(start)
}
func IsContextDone(ctx context.Context) bool {
select {
case <-ctx.Done():
return true
default:
}
return false
}
go-utils/bigint.go (1.8 KiB)
package goutils
import (
"github.com/shopspring/decimal"
"math/big"
)
// 加
func BigIntAdd(num1 string, num2 string) string {
x, _ := new(big.Int).SetString(num1, 10)
y, _ := new(big.Int).SetString(num2, 10)
x.Add(x, y)
return x.String()
}
// 减
func BigIntReduce(num1 string, num2 string) string {
x, _ := new(big.Int).SetString(num1, 10)
y, _ := new(big.Int).SetString("-"+num2, 10)
x.Add(x, y)
return x.String()
}
// 乘
func BigIntMul(num1 string, num2 string) string {
x, _ := new(big.Int).SetString(num1, 10)
y, _ := new(big.Int).SetString(num2, 10)
x.Mul(x, y)
return x.String()
}
// 除
func BigIntDiv(num1 string, num2 string) string {
x, _ := new(big.Int).SetString(num1, 10)
y, _ := new(big.Int).SetString(num2, 10)
x.Div(x, y)
return x.String()
}
// 取模
func BigIntMod(num1 string, num2 string) string {
x, _ := new(big.Int).SetString(num1, 10)
y, _ := new(big.Int).SetString(num2, 10)
x.Mod(x, y)
return x.String()
}
// 比大小,大于返回1,等于返回0,小于返回-1
func BigIntCmp(num1 string, num2 string) int {
x, _ := new(big.Int).SetString(num1, 10)
y, _ := new(big.Int).SetString(num2, 10)
return x.Cmp(y)
}
func SubFloat64(a float64, b float64) float64 {
_a := decimal.NewFromFloat(a)
_b := decimal.NewFromFloat(b)
_c, _ := _a.Sub(_b).Float64()
return _c
}
func MulFloat64(a float64, b float64) float64 {
_a := decimal.NewFromFloat(a)
_b := decimal.NewFromFloat(b)
_c, _ := _a.Mul(_b).Float64()
return _c
}
func AddFloat64(a float64, b float64) float64 {
_a := decimal.NewFromFloat(a)
_b := decimal.NewFromFloat(b)
_c, _ := _a.Add(_b).Float64()
return _c
}
func DivFloat64(a float64, b float64) float64 {
_a := decimal.NewFromFloat(a)
_b := decimal.NewFromFloat(b)
_c, _ := _a.Div(_b).Float64()
return _c
}
go-utils/captcha.go (2.0 KiB)
package goutils
import (
gocaptcha "github.com/gif-gif/go.io/go-captcha"
)
//https://github.com/mojocn/base64Captcha
//TODO: https://github.com/wenlng/go-captcha (高级)
var defaultCaptcha = gocaptcha.NewDefault()
func InitCaptcha(captcha *gocaptcha.GoCaptcha) {
defaultCaptcha = captcha
}
// 获取图片验证码-数字
func CaptchaGet(width, height int) map[string]string {
data, err := defaultCaptcha.DigitCaptcha(width, height, 4)
if err != nil {
return map[string]string{
"id": "",
"base64image": "",
}
}
return map[string]string{
"id": data.CaptchaId,
"base64image": data.Data,
}
}
func CaptchaChineseGet(width, height int, source string) map[string]string {
data, err := defaultCaptcha.ChineseCaptcha(width, height, 4, source)
if err != nil {
return map[string]string{
"id": "",
"base64image": "",
}
}
return map[string]string{
"id": data.CaptchaId,
"base64image": data.Data,
}
}
func CaptchaMathGet(width, height int) map[string]string {
data, err := defaultCaptcha.MathCaptcha(width, height)
if err != nil {
return map[string]string{
"id": "",
"base64image": "",
}
}
return map[string]string{
"id": data.CaptchaId,
"base64image": data.Data,
}
}
func CaptchaStringGet(width, height int) map[string]string {
if width == 0 {
width = 240
}
if height == 0 {
height = 80
}
data, err := defaultCaptcha.StringCaptcha(width, height, 4)
if err != nil {
return map[string]string{
"id": "",
"base64image": "",
}
}
return map[string]string{
"id": data.CaptchaId,
"base64image": data.Data,
}
}
func CaptchaAudioGet(audioStr string) map[string]string {
data, err := defaultCaptcha.AudioCaptcha(audioStr, 4)
if err != nil {
return map[string]string{
"id": "",
"base64image": "",
}
}
return map[string]string{
"id": data.CaptchaId,
"base64image": data.Data,
}
}
// 验证图片验证码
func CaptchaVerify(id, code string) bool {
return defaultCaptcha.CaptchaVerify(id, code)
}
go-utils/convert_type/converter.go (20.0 KiB)
package goconvert
import "time"
// String returns a pointer to the string value passed in.
func String(v string) *string {
return &v
}
// StringValue returns the value of the string pointer passed in or
// "" if the pointer is nil.
func StringValue(v *string) string {
if v != nil {
return *v
}
return ""
}
// StringSlice converts a slice of string values into a slice of
// string pointers
func StringSlice(src []string) []*string {
dst := make([]*string, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// StringValueSlice converts a slice of string pointers into a slice of
// string values
func StringValueSlice(src []*string) []string {
dst := make([]string, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// StringMap converts a string map of string values into a string
// map of string pointers
func StringMap(src map[string]string) map[string]*string {
dst := make(map[string]*string)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// StringValueMap converts a string map of string pointers into a string
// map of string values
func StringValueMap(src map[string]*string) map[string]string {
dst := make(map[string]string)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Bool returns a pointer to the bool value passed in.
func Bool(v bool) *bool {
return &v
}
// BoolValue returns the value of the bool pointer passed in or
// false if the pointer is nil.
func BoolValue(v *bool) bool {
if v != nil {
return *v
}
return false
}
// BoolSlice converts a slice of bool values into a slice of
// bool pointers
func BoolSlice(src []bool) []*bool {
dst := make([]*bool, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// BoolValueSlice converts a slice of bool pointers into a slice of
// bool values
func BoolValueSlice(src []*bool) []bool {
dst := make([]bool, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// BoolMap converts a string map of bool values into a string
// map of bool pointers
func BoolMap(src map[string]bool) map[string]*bool {
dst := make(map[string]*bool)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// BoolValueMap converts a string map of bool pointers into a string
// map of bool values
func BoolValueMap(src map[string]*bool) map[string]bool {
dst := make(map[string]bool)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Int returns a pointer to the int value passed in.
func Int(v int) *int {
return &v
}
// IntValue returns the value of the int pointer passed in or
// 0 if the pointer is nil.
func IntValue(v *int) int {
if v != nil {
return *v
}
return 0
}
// IntSlice converts a slice of int values into a slice of
// int pointers
func IntSlice(src []int) []*int {
dst := make([]*int, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// IntValueSlice converts a slice of int pointers into a slice of
// int values
func IntValueSlice(src []*int) []int {
dst := make([]int, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// IntMap converts a string map of int values into a string
// map of int pointers
func IntMap(src map[string]int) map[string]*int {
dst := make(map[string]*int)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// IntValueMap converts a string map of int pointers into a string
// map of int values
func IntValueMap(src map[string]*int) map[string]int {
dst := make(map[string]int)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Uint returns a pointer to the uint value passed in.
func Uint(v uint) *uint {
return &v
}
// UintValue returns the value of the uint pointer passed in or
// 0 if the pointer is nil.
func UintValue(v *uint) uint {
if v != nil {
return *v
}
return 0
}
// UintSlice converts a slice of uint values uinto a slice of
// uint pointers
func UintSlice(src []uint) []*uint {
dst := make([]*uint, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// UintValueSlice converts a slice of uint pointers uinto a slice of
// uint values
func UintValueSlice(src []*uint) []uint {
dst := make([]uint, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// UintMap converts a string map of uint values uinto a string
// map of uint pointers
func UintMap(src map[string]uint) map[string]*uint {
dst := make(map[string]*uint)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// UintValueMap converts a string map of uint pointers uinto a string
// map of uint values
func UintValueMap(src map[string]*uint) map[string]uint {
dst := make(map[string]uint)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Int8 returns a pointer to the int8 value passed in.
func Int8(v int8) *int8 {
return &v
}
// Int8Value returns the value of the int8 pointer passed in or
// 0 if the pointer is nil.
func Int8Value(v *int8) int8 {
if v != nil {
return *v
}
return 0
}
// Int8Slice converts a slice of int8 values into a slice of
// int8 pointers
func Int8Slice(src []int8) []*int8 {
dst := make([]*int8, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Int8ValueSlice converts a slice of int8 pointers into a slice of
// int8 values
func Int8ValueSlice(src []*int8) []int8 {
dst := make([]int8, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Int8Map converts a string map of int8 values into a string
// map of int8 pointers
func Int8Map(src map[string]int8) map[string]*int8 {
dst := make(map[string]*int8)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Int8ValueMap converts a string map of int8 pointers into a string
// map of int8 values
func Int8ValueMap(src map[string]*int8) map[string]int8 {
dst := make(map[string]int8)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Int16 returns a pointer to the int16 value passed in.
func Int16(v int16) *int16 {
return &v
}
// Int16Value returns the value of the int16 pointer passed in or
// 0 if the pointer is nil.
func Int16Value(v *int16) int16 {
if v != nil {
return *v
}
return 0
}
// Int16Slice converts a slice of int16 values into a slice of
// int16 pointers
func Int16Slice(src []int16) []*int16 {
dst := make([]*int16, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Int16ValueSlice converts a slice of int16 pointers into a slice of
// int16 values
func Int16ValueSlice(src []*int16) []int16 {
dst := make([]int16, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Int16Map converts a string map of int16 values into a string
// map of int16 pointers
func Int16Map(src map[string]int16) map[string]*int16 {
dst := make(map[string]*int16)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Int16ValueMap converts a string map of int16 pointers into a string
// map of int16 values
func Int16ValueMap(src map[string]*int16) map[string]int16 {
dst := make(map[string]int16)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Int32 returns a pointer to the int32 value passed in.
func Int32(v int32) *int32 {
return &v
}
// Int32Value returns the value of the int32 pointer passed in or
// 0 if the pointer is nil.
func Int32Value(v *int32) int32 {
if v != nil {
return *v
}
return 0
}
// Int32Slice converts a slice of int32 values into a slice of
// int32 pointers
func Int32Slice(src []int32) []*int32 {
dst := make([]*int32, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Int32ValueSlice converts a slice of int32 pointers into a slice of
// int32 values
func Int32ValueSlice(src []*int32) []int32 {
dst := make([]int32, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Int32Map converts a string map of int32 values into a string
// map of int32 pointers
func Int32Map(src map[string]int32) map[string]*int32 {
dst := make(map[string]*int32)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Int32ValueMap converts a string map of int32 pointers into a string
// map of int32 values
func Int32ValueMap(src map[string]*int32) map[string]int32 {
dst := make(map[string]int32)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Int64 returns a pointer to the int64 value passed in.
func Int64(v int64) *int64 {
return &v
}
// Int64Value returns the value of the int64 pointer passed in or
// 0 if the pointer is nil.
func Int64Value(v *int64) int64 {
if v != nil {
return *v
}
return 0
}
// Int64Slice converts a slice of int64 values into a slice of
// int64 pointers
func Int64Slice(src []int64) []*int64 {
dst := make([]*int64, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Int64ValueSlice converts a slice of int64 pointers into a slice of
// int64 values
func Int64ValueSlice(src []*int64) []int64 {
dst := make([]int64, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Int64Map converts a string map of int64 values into a string
// map of int64 pointers
func Int64Map(src map[string]int64) map[string]*int64 {
dst := make(map[string]*int64)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Int64ValueMap converts a string map of int64 pointers into a string
// map of int64 values
func Int64ValueMap(src map[string]*int64) map[string]int64 {
dst := make(map[string]int64)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Uint8 returns a pointer to the uint8 value passed in.
func Uint8(v uint8) *uint8 {
return &v
}
// Uint8Value returns the value of the uint8 pointer passed in or
// 0 if the pointer is nil.
func Uint8Value(v *uint8) uint8 {
if v != nil {
return *v
}
return 0
}
// Uint8Slice converts a slice of uint8 values into a slice of
// uint8 pointers
func Uint8Slice(src []uint8) []*uint8 {
dst := make([]*uint8, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Uint8ValueSlice converts a slice of uint8 pointers into a slice of
// uint8 values
func Uint8ValueSlice(src []*uint8) []uint8 {
dst := make([]uint8, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Uint8Map converts a string map of uint8 values into a string
// map of uint8 pointers
func Uint8Map(src map[string]uint8) map[string]*uint8 {
dst := make(map[string]*uint8)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Uint8ValueMap converts a string map of uint8 pointers into a string
// map of uint8 values
func Uint8ValueMap(src map[string]*uint8) map[string]uint8 {
dst := make(map[string]uint8)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Uint16 returns a pointer to the uint16 value passed in.
func Uint16(v uint16) *uint16 {
return &v
}
// Uint16Value returns the value of the uint16 pointer passed in or
// 0 if the pointer is nil.
func Uint16Value(v *uint16) uint16 {
if v != nil {
return *v
}
return 0
}
// Uint16Slice converts a slice of uint16 values into a slice of
// uint16 pointers
func Uint16Slice(src []uint16) []*uint16 {
dst := make([]*uint16, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Uint16ValueSlice converts a slice of uint16 pointers into a slice of
// uint16 values
func Uint16ValueSlice(src []*uint16) []uint16 {
dst := make([]uint16, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Uint16Map converts a string map of uint16 values into a string
// map of uint16 pointers
func Uint16Map(src map[string]uint16) map[string]*uint16 {
dst := make(map[string]*uint16)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Uint16ValueMap converts a string map of uint16 pointers into a string
// map of uint16 values
func Uint16ValueMap(src map[string]*uint16) map[string]uint16 {
dst := make(map[string]uint16)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Uint32 returns a pointer to the uint32 value passed in.
func Uint32(v uint32) *uint32 {
return &v
}
// Uint32Value returns the value of the uint32 pointer passed in or
// 0 if the pointer is nil.
func Uint32Value(v *uint32) uint32 {
if v != nil {
return *v
}
return 0
}
// Uint32Slice converts a slice of uint32 values into a slice of
// uint32 pointers
func Uint32Slice(src []uint32) []*uint32 {
dst := make([]*uint32, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Uint32ValueSlice converts a slice of uint32 pointers into a slice of
// uint32 values
func Uint32ValueSlice(src []*uint32) []uint32 {
dst := make([]uint32, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Uint32Map converts a string map of uint32 values into a string
// map of uint32 pointers
func Uint32Map(src map[string]uint32) map[string]*uint32 {
dst := make(map[string]*uint32)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Uint32ValueMap converts a string map of uint32 pointers into a string
// map of uint32 values
func Uint32ValueMap(src map[string]*uint32) map[string]uint32 {
dst := make(map[string]uint32)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Uint64 returns a pointer to the uint64 value passed in.
func Uint64(v uint64) *uint64 {
return &v
}
// Uint64Value returns the value of the uint64 pointer passed in or
// 0 if the pointer is nil.
func Uint64Value(v *uint64) uint64 {
if v != nil {
return *v
}
return 0
}
// Uint64Slice converts a slice of uint64 values into a slice of
// uint64 pointers
func Uint64Slice(src []uint64) []*uint64 {
dst := make([]*uint64, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Uint64ValueSlice converts a slice of uint64 pointers into a slice of
// uint64 values
func Uint64ValueSlice(src []*uint64) []uint64 {
dst := make([]uint64, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Uint64Map converts a string map of uint64 values into a string
// map of uint64 pointers
func Uint64Map(src map[string]uint64) map[string]*uint64 {
dst := make(map[string]*uint64)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Uint64ValueMap converts a string map of uint64 pointers into a string
// map of uint64 values
func Uint64ValueMap(src map[string]*uint64) map[string]uint64 {
dst := make(map[string]uint64)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Float32 returns a pointer to the float32 value passed in.
func Float32(v float32) *float32 {
return &v
}
// Float32Value returns the value of the float32 pointer passed in or
// 0 if the pointer is nil.
func Float32Value(v *float32) float32 {
if v != nil {
return *v
}
return 0
}
// Float32Slice converts a slice of float32 values into a slice of
// float32 pointers
func Float32Slice(src []float32) []*float32 {
dst := make([]*float32, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Float32ValueSlice converts a slice of float32 pointers into a slice of
// float32 values
func Float32ValueSlice(src []*float32) []float32 {
dst := make([]float32, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Float32Map converts a string map of float32 values into a string
// map of float32 pointers
func Float32Map(src map[string]float32) map[string]*float32 {
dst := make(map[string]*float32)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Float32ValueMap converts a string map of float32 pointers into a string
// map of float32 values
func Float32ValueMap(src map[string]*float32) map[string]float32 {
dst := make(map[string]float32)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Float64 returns a pointer to the float64 value passed in.
func Float64(v float64) *float64 {
return &v
}
// Float64Value returns the value of the float64 pointer passed in or
// 0 if the pointer is nil.
func Float64Value(v *float64) float64 {
if v != nil {
return *v
}
return 0
}
// Float64Slice converts a slice of float64 values into a slice of
// float64 pointers
func Float64Slice(src []float64) []*float64 {
dst := make([]*float64, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Float64ValueSlice converts a slice of float64 pointers into a slice of
// float64 values
func Float64ValueSlice(src []*float64) []float64 {
dst := make([]float64, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Float64Map converts a string map of float64 values into a string
// map of float64 pointers
func Float64Map(src map[string]float64) map[string]*float64 {
dst := make(map[string]*float64)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Float64ValueMap converts a string map of float64 pointers into a string
// map of float64 values
func Float64ValueMap(src map[string]*float64) map[string]float64 {
dst := make(map[string]float64)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Time returns a pointer to the time.Time value passed in.
func Time(v time.Time) *time.Time {
return &v
}
// TimeValue returns the value of the time.Time pointer passed in or
// time.Time{} if the pointer is nil.
func TimeValue(v *time.Time) time.Time {
if v != nil {
return *v
}
return time.Time{}
}
// SecondsTimeValue converts an int64 pointer to a time.Time value
// representing seconds since Epoch or time.Time{} if the pointer is nil.
func SecondsTimeValue(v *int64) time.Time {
if v != nil {
return time.Unix((*v / 1000), 0)
}
return time.Time{}
}
// MillisecondsTimeValue converts an int64 pointer to a time.Time value
// representing milliseconds sinch Epoch or time.Time{} if the pointer is nil.
func MillisecondsTimeValue(v *int64) time.Time {
if v != nil {
return time.Unix(0, (*v * 1000000))
}
return time.Time{}
}
// TimeUnixMilli returns a Unix timestamp in milliseconds from "January 1, 1970 UTC".
// The result is undefined if the Unix time cannot be represented by an int64.
// Which includes calling TimeUnixMilli on a zero Time is undefined.
//
// This utility is useful for service API's such as CloudWatch Logs which require
// their unix time values to be in milliseconds.
//
// See Go stdlib https://golang.org/pkg/time/#Time.UnixNano for more information.
func TimeUnixMilli(t time.Time) int64 {
return t.UnixNano() / int64(time.Millisecond/time.Nanosecond)
}
// TimeSlice converts a slice of time.Time values into a slice of
// time.Time pointers
func TimeSlice(src []time.Time) []*time.Time {
dst := make([]*time.Time, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// TimeValueSlice converts a slice of time.Time pointers into a slice of
// time.Time values
func TimeValueSlice(src []*time.Time) []time.Time {
dst := make([]time.Time, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// TimeMap converts a string map of time.Time values into a string
// map of time.Time pointers
func TimeMap(src map[string]time.Time) map[string]*time.Time {
dst := make(map[string]*time.Time)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// TimeValueMap converts a string map of time.Time pointers into a string
// map of time.Time values
func TimeValueMap(src map[string]*time.Time) map[string]time.Time {
dst := make(map[string]time.Time)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
go-utils/convert_type/utils.go (502 B)
package goconvert
import (
"encoding/json"
)
func ConvertToMap(data interface{}) map[string]interface{} {
var mapResult map[string]interface{}
dataBytes, _ := json.Marshal(data)
_ = json.Unmarshal(dataBytes, &mapResult)
return mapResult
}
func ConvertMapToStruct[T any](data map[string]interface{}) (*T, error) {
var t T
fieldsBytes, err := json.Marshal(data)
if err != nil {
return &t, err
}
err = json.Unmarshal(fieldsBytes, &t)
if err != nil {
return &t, err
}
return &t, nil
}
go-utils/convert_type/value_type.go (3.3 KiB)
package goconvert
import (
"encoding/json"
"fmt"
"github.com/gogf/gf/util/gconv"
)
const (
ValueTypeInt = "int"
ValueTypeString = "string"
ValueTypeText = "text"
ValueTypeStrings = "string[]"
ValueTypeBool = "bool"
ValueTypeDecimal = "decimal"
ValueTypeMap = "map"
ValueTypeJson = "json"
ValueTypeByte = "byte"
ValueTypeBytes = "bytes"
ValueTypeSelect = "select"
ValueTypeUnknown = "unknown"
)
// ConvertValue converts a string value to a specific type based on the provided valueType
func ConvertValue(value string, valueType string, result ...interface{}) (interface{}, error) {
switch valueType {
case ValueTypeInt:
return gconv.Int64(value), nil
case ValueTypeString:
return value, nil
case ValueTypeSelect:
return value, nil
case ValueTypeText:
return value, nil
case ValueTypeDecimal:
return gconv.Float64(value), nil
case ValueTypeBool:
return gconv.Bool(value), nil
case ValueTypeStrings:
return ConvertToStrings(value)
case ValueTypeByte:
return gconv.Byte(value), nil
case ValueTypeBytes:
return gconv.Bytes(value), nil
case ValueTypeMap:
return convertToMap(value)
case ValueTypeJson:
if len(result) == 0 {
return convertToJson(value)
}
return ConvertToJson(value, result[0])
case ValueTypeUnknown:
return value, nil
default:
return value, nil
}
}
// Helper functions for converting string to specific types
func ConvertToStrings(value string) ([]string, error) {
var result []string
err := json.Unmarshal([]byte(value), &result)
if err != nil {
return nil, fmt.Errorf("cannot convert string to []string: %w", err)
}
return result, nil
}
func convertToMap(value string) (map[string]interface{}, error) {
var result map[string]interface{}
err := json.Unmarshal([]byte(value), &result)
if err != nil {
return nil, fmt.Errorf("cannot convert string to map: %w", err)
}
return result, nil
}
func convertToJson(value string) (interface{}, error) {
var result interface{}
return ConvertToJson(value, result)
}
func ConvertToJson(value string, result interface{}) (interface{}, error) {
err := json.Unmarshal([]byte(value), &result)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
}
return result, nil
}
// Example usage
func main() {
// 测试一些字符串转换的例子
intStr := "42"
floatStr := "3.14"
stringVal := "hello"
mapStr := `{"name":"John","age":30}`
jsonStr := `{"name":"John","age":30,"address":{"city":"New York","zip":"10001"}}`
intResult, err := ConvertValue(intStr, ValueTypeInt)
fmt.Printf("Int64: %v, %T, Error: %v\n", intResult, intResult, err)
floatResult, err := ConvertValue(floatStr, ValueTypeDecimal)
fmt.Printf("Float32: %v, %T, Error: %v\n", floatResult, floatResult, err)
stringResult, err := ConvertValue(stringVal, ValueTypeString)
fmt.Printf("String: %v, %T, Error: %v\n", stringResult, stringResult, err)
byteResult, err := ConvertValue(intStr, ValueTypeByte)
fmt.Printf("Byte: %v, %T, Error: %v\n", byteResult, byteResult, err)
bytesResult, err := ConvertValue(stringVal, ValueTypeBytes)
fmt.Printf("Bytes: %v, %T, Error: %v\n", bytesResult, bytesResult, err)
mapResult, err := ConvertValue(mapStr, ValueTypeMap)
fmt.Printf("Map: %v, %T, Error: %v\n", mapResult, mapResult, err)
jsonResult, err := ConvertValue(jsonStr, ValueTypeJson)
fmt.Printf("JSON: %v, %T, Error: %v\n", jsonResult, jsonResult, err)
}
go-utils/crypto.go (8.5 KiB)
package goutils
import (
"bytes"
"crypto"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/md5"
"crypto/rand"
"crypto/rsa"
"crypto/sha1"
"crypto/sha256"
"crypto/x509"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"io"
"math/big"
"net/url"
"os"
"strings"
)
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 生成 AES 密钥
func GenerateAESKey() (string, error) {
// 生成32字节(256位)的密钥
key := make([]byte, 32)
_, err := rand.Read(key)
if err != nil {
return "", err
}
return hex.EncodeToString(key), nil
}
// 生成 AES 密钥和 IV
func GenerateAESKeyAndIV() (string, string, error) {
// 生成 16 字节(128 位)的 Key
key := make([]byte, 16)
_, err := rand.Read(key)
if err != nil {
return "", "", err
}
// 生成 16 字节(128 位)的 IV
iv := make([]byte, 32)
_, err = rand.Read(iv)
if err != nil {
return "", "", err
}
return hex.EncodeToString(key), hex.EncodeToString(iv), nil
}
// 计算文件md5(支持超大文件)
func CalculateFileMD5(filePath string) (string, error) {
// 打开文件
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()
// 创建MD5哈希对象
hash := md5.New()
// 创建一个缓冲区,逐块读取文件内容
buffer := make([]byte, 1024*1024) // 1MB 缓冲区
for {
n, err := file.Read(buffer)
if err != nil && err != io.EOF {
return "", err
}
if n == 0 {
break
}
// 更新哈希值
if _, err := hash.Write(buffer[:n]); err != nil {
return "", err
}
}
// 计算最终的哈希值
hashInBytes := hash.Sum(nil)
hashInString := fmt.Sprintf("%x", hashInBytes)
return hashInString, nil
}
// MD5 大写
func MD5(buf []byte) string {
h := md5.New()
h.Write(buf)
return strings.ToUpper(hex.EncodeToString(h.Sum(nil)))
}
// Md5小写
func Md5(buf []byte) string {
h := md5.New()
h.Write(buf)
return strings.ToLower(hex.EncodeToString(h.Sum(nil)))
}
// 小写
func SHA1(buf []byte) string {
h := sha1.New()
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func SHA256(buf, key []byte) string {
h := hmac.New(sha256.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func HMacMd5(buf, key []byte) string {
h := hmac.New(md5.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func HMacSha1(buf, key []byte) string {
h := hmac.New(sha1.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func HMacSha256(buf, key []byte) string {
h := hmac.New(sha256.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func Base64Encode(buf []byte) string {
return base64.StdEncoding.EncodeToString(buf)
}
func Base64Decode(str string) []byte {
var count = (4 - len(str)%4) % 4
str += strings.Repeat("=", count)
buf, _ := base64.StdEncoding.DecodeString(str)
return buf
}
func SHAWithRSA(key, data []byte) (string, error) {
pkey, err := x509.ParsePKCS8PrivateKey(key)
if err != nil {
return "", err
}
h := crypto.Hash.New(crypto.SHA1)
h.Write(data)
hashed := h.Sum(nil)
buf, err := rsa.SignPKCS1v15(rand.Reader, pkey.(*rsa.PrivateKey), crypto.SHA1, hashed)
if err != nil {
return "", err
}
return Base64Encode(buf), nil
}
func AESECBEncrypt(data, key []byte) ([]byte, error) {
cb, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
blockSize := cb.BlockSize()
paddingSize := blockSize - len(data)%blockSize
if paddingSize != 0 {
data = append(data, bytes.Repeat([]byte{byte(0)}, paddingSize)...)
}
encrypted := make([]byte, len(data))
for bs, be := 0, blockSize; bs < len(data); bs, be = bs+blockSize, be+blockSize {
cb.Encrypt(encrypted[bs:be], data[bs:be])
}
return encrypted, nil
}
func AESECBDecrypt(buf, key []byte) ([]byte, error) {
cb, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
blockSize := cb.BlockSize()
decrypted := make([]byte, len(buf))
for bs, be := 0, blockSize; bs < len(buf); bs, be = bs+blockSize, be+blockSize {
cb.Decrypt(decrypted[bs:be], buf[bs:be])
}
paddingSize := int(decrypted[len(decrypted)-1])
return decrypted[0 : len(decrypted)-paddingSize], nil
}
func AESCBCEncrypt(rawData, key, iv []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
// block 大小 16
blockSize := block.BlockSize()
// 填充原文
rawData = pkcs7padding(rawData, blockSize)
// 定义密码数据
var cipherData []byte
// 如果iv为空,生成随机iv,并附加到加密数据前面,否则单独生成加密数据
if iv == nil {
// 初始化加密数据
cipherData = make([]byte, blockSize+len(rawData))
// 定义向量
iv = cipherData[:blockSize]
// 填充向量IV, ReadFull从rand.Reader精确地读取len(b)字节数据填充进iv,rand.Reader是一个全局、共享的密码用强随机数生成器
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
return nil, err
}
// 加密
mode := cipher.NewCBCEncrypter(block, iv)
mode.CryptBlocks(cipherData[blockSize:], rawData)
} else {
// 初始化加密数据
cipherData = make([]byte, len(rawData))
// 定义向量
iv = iv[:blockSize]
// 加密
mode := cipher.NewCBCEncrypter(block, iv)
mode.CryptBlocks(cipherData, rawData)
}
return cipherData, nil
}
func AESCBCDecrypt(cipherData, key, iv []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
// block 大小 16
blockSize := block.BlockSize()
// 加密串长度
l := len(cipherData)
// 校验长度
if l < blockSize {
return nil, errors.New("encrypt data too short")
}
// 定义原始数据
var origData []byte
// 如果iv为空,需要获取前16位作为随机iv
if iv == nil {
// 定义向量
iv = cipherData[:blockSize]
// 定义真实加密串
cipherData = cipherData[blockSize:]
// 初始化原始数据
origData = make([]byte, l-blockSize)
} else {
// 定义向量
iv = iv[:blockSize]
// 初始化原始数据
origData = make([]byte, l)
}
// 解密
mode := cipher.NewCBCDecrypter(block, iv)
mode.CryptBlocks(origData, cipherData)
origData = pkcs7unpadding(origData)
return origData, nil
}
func pkcs7padding(cipherText []byte, blockSize int) []byte {
padding := blockSize - len(cipherText)%blockSize
padText := bytes.Repeat([]byte{byte(padding)}, padding)
return append(cipherText, padText...)
}
func pkcs7unpadding(origData []byte) []byte {
l := len(origData)
unPadding := int(origData[l-1])
if l < unPadding {
return nil
}
return origData[:(l - unPadding)]
}
func SessionId() string {
buf := make([]byte, 32)
_, err := rand.Read(buf)
if err != nil {
return ""
}
return hex.EncodeToString(buf)
}
const (
base59key = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ."
)
// 如果遇到特殊字符,需要用 url.PathEscape(str) 解决
func Base59Encoding(strByte []byte, key ...string) string {
strByte = []byte(url.PathEscape(string(strByte)))
if l := len(key); l == 0 || key[0] == "" {
key = []string{base59key}
}
base := int64(59)
strTen := big.NewInt(0).SetBytes(strByte)
keyByte := []byte(key[0])
var modSlice []byte
for strTen.Cmp(big.NewInt(0)) > 0 {
mod := big.NewInt(0)
strTen5 := big.NewInt(base)
strTen.DivMod(strTen, strTen5, mod)
modSlice = append(modSlice, keyByte[mod.Int64()])
}
for _, elem := range strByte {
if elem != 0 {
break
}
if elem == 0 {
modSlice = append(modSlice, byte('1'))
}
}
ReverseModSlice := reverseByteArr(modSlice)
return string(ReverseModSlice)
}
func reverseByteArr(bytes []byte) []byte {
for i := 0; i < len(bytes)/2; i++ {
bytes[i], bytes[len(bytes)-1-i] = bytes[len(bytes)-1-i], bytes[i]
}
return bytes
}
func Base59Decoding(strByte []byte, key ...string) []byte {
if l := len(key); l == 0 || key[0] == "" {
key = []string{base59key}
}
base := int64(59)
ret := big.NewInt(0)
for _, byteElem := range strByte {
index := bytes.IndexByte([]byte(key[0]), byteElem)
ret.Mul(ret, big.NewInt(base))
ret.Add(ret, big.NewInt(int64(index)))
}
str, _ := url.PathUnescape(string(ret.Bytes()))
return []byte(str)
}
func UrlEncode(str string) string {
return url.QueryEscape(str)
}
func UrlDecode(str string) string {
str, _ = url.QueryUnescape(str)
return str
}
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
// 这个包即将废弃 用gocrypto包
go-utils/email.go (702 B)
package goutils
import (
"net/mail"
"regexp"
)
func HideEmail(email string) string {
re := regexp.MustCompile("(?P<name>[^@]+)@(?P<domain>[^@]+\\.[^@]+)")
matches := re.FindStringSubmatch(email)
if len(matches) < 3 {
return email
}
// 隐藏用户名的一部分
name := matches[1]
hiddenName := ""
if len(name) > 3 {
// 前三个字符保持不变,后面的字符替换为星号
hiddenName = name[:3] + string(make([]rune, len(name)-3, len(name)-3))
} else {
hiddenName = string(make([]rune, len(name), len(name))) // 全部替换为星号
}
return hiddenName + "@" + matches[2]
}
func IsEmail(email string) bool {
_, err := mail.ParseAddress(email)
return err == nil
}
go-utils/encoding.go (732 B)
package goutils
import (
"bytes"
golog "github.com/gif-gif/go.io/go-log"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/transform"
"io/ioutil"
)
func GBK2UTF8(s string) string {
r := bytes.NewBufferString(s)
rr := transform.NewReader(r, simplifiedchinese.GBK.NewDecoder())
buf, err := ioutil.ReadAll(rr)
if err != nil {
golog.WithField("str", s).Error(err.Error())
return ""
}
return string(bytes.TrimSpace(buf))
}
func UTF82GBK(s string) string {
r := bytes.NewBufferString(s)
rr := transform.NewReader(r, simplifiedchinese.GBK.NewEncoder())
buf, err := ioutil.ReadAll(rr)
if err != nil {
golog.WithField("str", s).Error(err.Error())
return ""
}
return string(bytes.TrimSpace(buf))
}
go-utils/gobean/bean.go (3.1 KiB)
package gobean
import (
"encoding/json"
"fmt"
"golang.org/x/exp/constraints"
)
const (
OpEquals = "equals"
OpLessOrEquals = "lessEqual"
OpGreaterOrEqual = "greaterEqual"
OpGreater = "greater"
OpLess = "less"
)
type NumberRange[T any] struct {
Min *T `json:"min,optional"`
Max *T `json:"max,optional"`
}
type Int64Range struct {
Min *int64 `json:"min,optional"`
Max *int64 `json:"max,optional"`
}
type Float64Range struct {
Min *float64 `json:"min,optional"`
Max *float64 `json:"max,optional"`
}
type Int64Method struct {
Method string `json:"method,optional"`
Value int64 `json:"value,optional"`
}
func (r *Int64Method) Check(val int64) bool {
return CheckValue(r, val)
}
func (r *Int64Method) GetValue() int64 {
return r.Value
}
func (r *Int64Method) GetMethod() string {
return r.Method
}
type IOperationMethod[T any] interface {
GetMethod() string
GetValue() T
}
// 大于,大于等于,小于,小于等于,等于
func CheckValue(r IOperationMethod[int64], val int64) bool {
method := r.GetMethod()
value := r.GetValue()
switch method {
case OpEquals:
return val == value
case OpLessOrEquals:
return val <= value
case OpGreaterOrEqual:
return val >= value
case OpGreater:
return val > value
case OpLess:
return val < value
}
return false
}
// IsInRange 检查给定值是否在指定的范围内
//
// 如果 Min 为 nil,则没有下限限制
//
// 如果 Max 为 nil,则没有上限限制
//
// 如果 Min 和 Max 都为 nil,则认为没有限制,任何值都符合条件
func IsInRange[T constraints.Ordered](value T, rng NumberRange[T]) bool {
// 如果 Min 和 Max 都为 nil,则没有限制
if rng.Min == nil && rng.Max == nil {
return true
}
// 检查下限
if rng.Min != nil && value < *rng.Min {
return false
}
// 检查上限
if rng.Max != nil && value > *rng.Max {
return false
}
return true
}
// CheckRanges 检查给定值是否满足所有范围条件
//
// 有符合条件的返回true,否则false
func CheckRanges[T constraints.Ordered](value T, ranges []NumberRange[T]) bool {
// 如果没有条件,则视为没有限制
if len(ranges) == 0 {
return false
}
// 检查是否满足所有条件
for _, rng := range ranges {
if IsInRange(value, rng) {
return true
}
}
return false
}
// 创建一个帮助函数,用于更方便地创建 NumberRange
func NewNumberRange[T constraints.Ordered](min, max *T) NumberRange[T] {
return NumberRange[T]{
Min: min,
Max: max,
}
}
// 有符合条件的返回true,否则false
func CheckValueAgainstRanges[T constraints.Ordered](value T, jsonRanges string) (bool, error) {
// 如果 JSON 字符串为空,视为没有限制
if jsonRanges == "" {
return false, nil
}
// 解析 JSON 数组
var ranges []NumberRange[T]
err := json.Unmarshal([]byte(jsonRanges), &ranges)
if err != nil {
return false, fmt.Errorf("解析 JSON 失败: %w", err)
}
// 空数组视为没有限制
if len(ranges) == 0 {
return false, nil
}
// 检查是否满足所有条件
for _, rng := range ranges {
if IsInRange(value, rng) {
return true, nil
}
}
return false, nil
}
go-utils/gobean/constants.go (694 B)
package gobean
const (
GoogleIDEmpty = "00000000-0000-0000-0000-000000000000"
//platform
PlatformMobileIos = "mobile-ios"
PlatformMobileAndroid = "mobile-android"
PlatformDesktopWindows = "desktop-windows"
PlatformDesktopMacOs = "desktop-macos"
//channel
ChannelApple = "apple"
ChannelGoogle = "google"
ChannelUniversal = "universal"
)
func CheckPlatform(platform string) bool {
return platform == PlatformMobileIos || platform == PlatformMobileAndroid || platform == PlatformDesktopWindows || platform == PlatformDesktopMacOs
}
func CheckChannel(channel string) bool {
return channel == ChannelApple || channel == ChannelGoogle || channel == ChannelUniversal
}
go-utils/gobean/readme.md (26 B)
常用 golang 结构体
go-utils/gocrypto/crypto.go (12.3 KiB)
package gocrypto
import (
"bytes"
"crypto"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/md5"
"crypto/rand"
"crypto/rsa"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"crypto/x509"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"github.com/gogf/gf/util/gconv"
"golang.org/x/crypto/bcrypt"
"hash"
"io"
"math"
"math/big"
"net/url"
"os"
"strings"
"time"
)
//md5
//sha1
//sha256
//sha512
const (
HashingAlgorithmMd5 = "md5"
HashingAlgorithmSha1 = "sha1"
HashingAlgorithmSha256 = "sha256"
HashingAlgorithmSha512 = "sha512"
)
//hex(十六进制)
//base64
const (
EncodingHex = "hex"
EncodingBase64 = "base64"
)
// CreateHash 计算文件的哈希值
// data: 输入数据
// hashingAlgorithm: 哈希算法名称 ("md5", "sha1", "sha256", "sha512" 等)
// encoding: 编码方式 ("hex" 或 "base64")
func CreateHash(data []byte, hashingAlgorithm string, encoding string) (string, error) {
// 选择哈希算法
var h hash.Hash
switch hashingAlgorithm {
case HashingAlgorithmMd5:
h = md5.New()
case HashingAlgorithmSha1:
h = sha1.New()
case HashingAlgorithmSha256:
h = sha256.New()
case HashingAlgorithmSha512:
h = sha512.New()
default:
return "", fmt.Errorf("unsupported hashing algorithm: %s", hashingAlgorithm)
}
// 写入数据
h.Write(data)
// 获取哈希值
hashBytes := h.Sum(nil)
// 根据指定编码格式返回结果
switch encoding {
case EncodingHex:
return hex.EncodeToString(hashBytes), nil
case EncodingBase64:
return base64.StdEncoding.EncodeToString(hashBytes), nil
default:
return "", fmt.Errorf("unsupported encoding: %s", encoding)
}
}
func GenerateByteKey(len int64) ([]byte, error) {
// 生成32字节(256位)的密钥
key := make([]byte, len)
_, err := rand.Read(key)
if err != nil {
return nil, err
}
return key, nil
}
// 生成随机字符串,可用于生成随机密钥, len 为长度
func GenerateKey(len int64) (string, error) {
// 生成32字节(256位)的密钥
key := make([]byte, len)
_, err := rand.Read(key)
if err != nil {
return "", err
}
return hex.EncodeToString(key), nil
}
// 生成 AES 密钥
func GenerateAESKey() (string, error) {
// 生成32字节(256位)的密钥
key, err := GenerateKey(32)
if err != nil {
return "", err
}
return key, nil
}
func GenerateAESIv() (string, error) {
// 生成 16 字节(128 位)的 IV
key, err := GenerateKey(16)
if err != nil {
return "", err
}
return key, nil
}
// 生成 AES 密钥和 IV 返回string 格式(hex.EncodeToString)
func GenerateAESKeyAndIV() (string, string, error) {
// 生成32字节(256位)的密钥
key, err := GenerateKey(32)
if err != nil {
return "", "", err
}
// 生成 16 字节(128 位)的 IV
iv, err := GenerateKey(16)
if err != nil {
return "", "", err
}
return key, iv, nil
}
// 生成 AES 密钥和 IV 返回string 格式(hex.EncodeToString)
func GenerateAESKeyAndIVBase64() (string, string, error) {
// 生成32字节(256位)的密钥
key, err := GenerateByteKey(32)
if err != nil {
return "", "", err
}
// 生成 16 字节(128 位)的 IV
iv, err := GenerateByteKey(16)
if err != nil {
return "", "", err
}
return Base64EncodeString(key), Base64EncodeString(iv), nil
}
// 计算文件md5(支持超大文件)
func CalculateFileMD5(filePath string) (string, error) {
// 打开文件
file, err := os.Open(filePath)
if err != nil {
return "", err
}
defer file.Close()
// 创建MD5哈希对象
hash := md5.New()
// 创建一个缓冲区,逐块读取文件内容
buffer := make([]byte, 1024*1024) // 1MB 缓冲区
for {
n, err := file.Read(buffer)
if err != nil && err != io.EOF {
return "", err
}
if n == 0 {
break
}
// 更新哈希值
if _, err := hash.Write(buffer[:n]); err != nil {
return "", err
}
}
// 计算最终的哈希值
hashInBytes := hash.Sum(nil)
hashInString := fmt.Sprintf("%x", hashInBytes)
return hashInString, nil
}
// MD5 大写
func MD5(buf []byte) string {
h := md5.New()
h.Write(buf)
return strings.ToUpper(hex.EncodeToString(h.Sum(nil)))
}
// Md5小写
func Md5(buf []byte) string {
h := md5.New()
h.Write(buf)
return strings.ToLower(hex.EncodeToString(h.Sum(nil)))
}
// 小写
func SHA1(buf []byte) string {
h := sha1.New()
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func SHA256(buf, key []byte) string {
h := hmac.New(sha256.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func HMacMd5(buf, key []byte) string {
h := hmac.New(md5.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func HMacSha1(buf, key []byte) string {
h := hmac.New(sha1.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func HMacSha256(buf, key []byte) string {
h := hmac.New(sha256.New, key)
h.Write(buf)
return hex.EncodeToString(h.Sum(nil))
}
func Base64EncodeString(buf []byte) string {
return base64.StdEncoding.EncodeToString(buf)
}
func Base64DecodeString(str string) []byte {
var count = (4 - len(str)%4) % 4
str += strings.Repeat("=", count)
buf, _ := base64.StdEncoding.DecodeString(str)
return buf
}
func Base64Encode(buf []byte) []byte {
dBuf := make([]byte, base64.StdEncoding.DecodedLen(len(buf)))
n, err := base64.StdEncoding.Decode(dBuf, buf)
if err != nil {
return nil
}
return dBuf[:n]
}
func Base64Decode(buf []byte) []byte {
dBuf := make([]byte, base64.StdEncoding.DecodedLen(len(buf)))
_, err := base64.StdEncoding.Decode(dBuf, buf)
if err != nil {
return nil
}
return dBuf
}
func SHAWithRSA(key, data []byte) (string, error) {
pkey, err := x509.ParsePKCS8PrivateKey(key)
if err != nil {
return "", err
}
h := crypto.Hash.New(crypto.SHA1)
h.Write(data)
hashed := h.Sum(nil)
buf, err := rsa.SignPKCS1v15(rand.Reader, pkey.(*rsa.PrivateKey), crypto.SHA1, hashed)
if err != nil {
return "", err
}
return Base64EncodeString(buf), nil
}
// AesCTRTransform 使用 AES CTR 模式对数据进行加密或解密
// 由于 CTR 模式的特性,加密和解密操作是相同的
func AesCTRTransform(data, key, iv []byte) ([]byte, error) {
// 参数验证
if len(data) <= 0 {
return nil, errors.New("输入数据为空")
}
if len(key) <= 0 {
return nil, errors.New("密钥为空")
}
if len(iv) <= 0 || len(iv) < aes.BlockSize {
return nil, errors.New("初始化向量为空或长度不足")
}
// 创建 AES 密码块
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
// 创建输出缓冲区
output := make([]byte, len(data))
// 创建 CTR 模式的加密器
stream := cipher.NewCTR(block, iv)
// 执行加密/解密操作
// 在 CTR 模式中,XorKeyStream 既可用于加密也可用于解密
stream.XORKeyStream(output, data)
return output, nil
}
func AESECBEncrypt(data, key []byte) ([]byte, error) {
cb, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
blockSize := cb.BlockSize()
paddingSize := blockSize - len(data)%blockSize
if paddingSize != 0 {
data = append(data, bytes.Repeat([]byte{byte(0)}, paddingSize)...)
}
encrypted := make([]byte, len(data))
for bs, be := 0, blockSize; bs < len(data); bs, be = bs+blockSize, be+blockSize {
cb.Encrypt(encrypted[bs:be], data[bs:be])
}
return encrypted, nil
}
func AESECBDecrypt(buf, key []byte) ([]byte, error) {
cb, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
blockSize := cb.BlockSize()
decrypted := make([]byte, len(buf))
for bs, be := 0, blockSize; bs < len(buf); bs, be = bs+blockSize, be+blockSize {
cb.Decrypt(decrypted[bs:be], buf[bs:be])
}
paddingSize := int(decrypted[len(decrypted)-1])
return decrypted[0 : len(decrypted)-paddingSize], nil
}
func AESCBCEncrypt(rawData, key, iv []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
// block 大小 16
blockSize := block.BlockSize()
// 填充原文
rawData = pkcs7padding(rawData, blockSize)
// 定义密码数据
var cipherData []byte
// 如果iv为空,生成随机iv,并附加到加密数据前面,否则单独生成加密数据
if iv == nil {
// 初始化加密数据
cipherData = make([]byte, blockSize+len(rawData))
// 定义向量
iv = cipherData[:blockSize]
// 填充向量IV, ReadFull从rand.Reader精确地读取len(b)字节数据填充进iv,rand.Reader是一个全局、共享的密码用强随机数生成器
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
return nil, err
}
// 加密
mode := cipher.NewCBCEncrypter(block, iv)
mode.CryptBlocks(cipherData[blockSize:], rawData)
} else {
// 初始化加密数据
cipherData = make([]byte, len(rawData))
// 定义向量
iv = iv[:blockSize]
// 加密
mode := cipher.NewCBCEncrypter(block, iv)
mode.CryptBlocks(cipherData, rawData)
}
return cipherData, nil
}
func AESCBCDecrypt(cipherData, key, iv []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
// block 大小 16
blockSize := block.BlockSize()
// 加密串长度
l := len(cipherData)
// 校验长度
if l < blockSize {
return nil, errors.New("encrypt data too short")
}
// 定义原始数据
var origData []byte
// 如果iv为空,需要获取前16位作为随机iv
if iv == nil {
// 定义向量
iv = cipherData[:blockSize]
// 定义真实加密串
cipherData = cipherData[blockSize:]
// 初始化原始数据
origData = make([]byte, l-blockSize)
} else {
// 定义向量
iv = iv[:blockSize]
// 初始化原始数据
origData = make([]byte, l)
}
// 解密
mode := cipher.NewCBCDecrypter(block, iv)
mode.CryptBlocks(origData, cipherData)
origData = pkcs7unpadding(origData)
return origData, nil
}
func pkcs7padding(cipherText []byte, blockSize int) []byte {
padding := blockSize - len(cipherText)%blockSize
padText := bytes.Repeat([]byte{byte(padding)}, padding)
return append(cipherText, padText...)
}
func pkcs7unpadding(origData []byte) []byte {
l := len(origData)
unPadding := int(origData[l-1])
if l < unPadding {
return nil
}
return origData[:(l - unPadding)]
}
func SessionId() string {
buf := make([]byte, 32)
_, err := rand.Read(buf)
if err != nil {
return ""
}
return hex.EncodeToString(buf)
}
const (
base59key = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ."
)
// 如果遇到特殊字符,需要用 url.PathEscape(str) 解决
func Base59Encoding(strByte []byte, key ...string) string {
strByte = []byte(url.PathEscape(string(strByte)))
if l := len(key); l == 0 || key[0] == "" {
key = []string{base59key}
}
base := int64(59)
strTen := big.NewInt(0).SetBytes(strByte)
keyByte := []byte(key[0])
var modSlice []byte
for strTen.Cmp(big.NewInt(0)) > 0 {
mod := big.NewInt(0)
strTen5 := big.NewInt(base)
strTen.DivMod(strTen, strTen5, mod)
modSlice = append(modSlice, keyByte[mod.Int64()])
}
for _, elem := range strByte {
if elem != 0 {
break
}
if elem == 0 {
modSlice = append(modSlice, byte('1'))
}
}
ReverseModSlice := reverseByteArr(modSlice)
return string(ReverseModSlice)
}
func reverseByteArr(bytes []byte) []byte {
for i := 0; i < len(bytes)/2; i++ {
bytes[i], bytes[len(bytes)-1-i] = bytes[len(bytes)-1-i], bytes[i]
}
return bytes
}
func Base59Decoding(strByte []byte, key ...string) []byte {
if l := len(key); l == 0 || key[0] == "" {
key = []string{base59key}
}
base := int64(59)
ret := big.NewInt(0)
for _, byteElem := range strByte {
index := bytes.IndexByte([]byte(key[0]), byteElem)
ret.Mul(ret, big.NewInt(base))
ret.Add(ret, big.NewInt(int64(index)))
}
str, _ := url.PathUnescape(string(ret.Bytes()))
return []byte(str)
}
func UrlEncode(str string) string {
return url.QueryEscape(str)
}
func UrlDecode(str string) string {
str, _ = url.QueryUnescape(str)
return str
}
// BcryptHash 使用 bcrypt 对密码进行加密
func BcryptHash(password string) string {
bytes, _ := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
return string(bytes)
}
// BcryptCheck 对比明文密码和数据库的哈希值
func BcryptCheck(password, hash string) bool {
err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password))
return err == nil
}
// 常用签名验证, sign md5 小写
func CheckSign(secret string, linkSignTimeout int64, ts int64, sign string) bool {
if linkSignTimeout == 0 {
linkSignTimeout = 20
}
tsStep := time.Now().Unix() - ts
if math.Abs(gconv.Float64(tsStep)) > gconv.Float64(linkSignTimeout) { //连接失效
return false
}
serverSign := Md5([]byte(gconv.String(ts) + secret))
return serverSign == sign
}
go-utils/gocrypto/utils.go (4.1 KiB)
package gocrypto
import (
"encoding/binary"
goerror "github.com/gif-gif/go.io/go-error"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gif-gif/go.io/go-utils/gozip"
"time"
)
// AesIv 动态生成 (aes(time+zip(data)))
//
// 1. 先时间戳 time
//
// 2. 生成16位随机IV
//
// 3. 压缩data(如果有压缩) zip(data)
//
// 4. 加密data: aes(time+zip(data))
//
// 5. 拼接iv+encryptData
func GoDataEncrypt(data []byte, AesKey []byte, compressMethod string) ([]byte, error) {
defer goutils.Recovery(func(err any) {
golog.Warn(err)
})
timestamp := time.Now().Unix()
timestampBytes := make([]byte, 8)
binary.BigEndian.PutUint64(timestampBytes, uint64(timestamp))
randomIv, err := GenerateByteKey(16)
if err != nil {
return nil, err
}
var compressBytes []byte
if compressMethod != "" && compressMethod != gozip.NOZIP {
_, compressBytes, err = gozip.Compress(data, compressMethod, gozip.GoZipType)
if err != nil {
return nil, err
}
} else {
compressBytes = data
}
timeAndData := append(timestampBytes, compressBytes...)
dataEncrypt, err := AESCBCEncrypt(timeAndData, AesKey, randomIv)
if err != nil {
return nil, err
}
resDataEncrypt := append(randomIv, dataEncrypt...)
return resDataEncrypt, nil
}
// 解密:
//
// 1. 先取前16个字节,作为AES的IV
//
// 2. 取剩余的字节解密
//
// 3. 取前8个字节,作为时间戳
//
// 4. 取剩余的字节
//
// 5. 解压data(如果有压缩)
func GoDataDecrypt(data []byte, AesKey []byte, compressMethod string) ([]byte, error) {
defer goutils.Recovery(func(err any) {
golog.Warn(err)
})
AesIvLength := 16
if len(data) < AesIvLength { //非法数据
return nil, goerror.NewParamErrMsg("非法数据")
}
first16BytesIv := data[:AesIvLength]
// 获取剩余的字节解密
timeAndDataEncryptBytes := data[AesIvLength:]
timeAndZipBody, err := AESCBCDecrypt(timeAndDataEncryptBytes, AesKey, first16BytesIv)
if err != nil {
return nil, err
}
if len(timeAndZipBody) < 8 { //非法数据
return nil, goerror.NewParamErrMsg("非法数据")
}
timestampBytes := timeAndZipBody[:8]
_ = binary.BigEndian.Uint64(timestampBytes) //客户端时间
body := timeAndZipBody[8:]
if compressMethod != "" && compressMethod != gozip.NOZIP {
_, body, err = gozip.Compress(body, compressMethod, gozip.UnGoZipType)
if err != nil {
return nil, err
}
}
return body, nil
}
// 加密和解密AesCtr(zip(data))
//
// compressMethod 空时不会压缩和解压
func GoDataAesCTRTransformEncode(data []byte, aesKey []byte, aesIv []byte, compressMethod string) ([]byte, error) {
defer goutils.Recovery(func(err any) {
golog.Warn(err)
})
var compressBytes []byte
var err error
if compressMethod != "" && compressMethod != gozip.NOZIP {
_, compressBytes, err = gozip.Compress(data, compressMethod, gozip.GoZipType)
if err != nil {
return nil, err
}
} else {
compressBytes = data
}
body, err := AesCTRTransform(compressBytes, aesKey, aesIv)
if err != nil {
return nil, err
}
return body, nil
}
func GoDataAesCTRTransformDecode(data []byte, aesKey []byte, aesIv []byte, compressMethod string) ([]byte, error) {
defer goutils.Recovery(func(err any) {
golog.Warn(err)
})
var err error
body, err := AesCTRTransform(data, aesKey, aesIv)
if err != nil {
return nil, err
}
var compressBytes []byte
if compressMethod != "" && compressMethod != gozip.NOZIP {
_, compressBytes, err = gozip.Compress(body, compressMethod, gozip.UnGoZipType)
if err != nil {
return nil, err
}
} else {
compressBytes = body
}
return compressBytes, nil
}
func GoDataAesCTRTransform(data []byte, aesKey []byte, aesIv []byte, compressMethod string) ([]byte, error) {
defer goutils.Recovery(func(err any) {
golog.Warn(err)
})
var err error
body, err := AesCTRTransform(data, aesKey, aesIv)
if err != nil {
return nil, err
}
var compressBytes []byte
if compressMethod != "" && compressMethod != gozip.NOZIP {
_, compressBytes, err = gozip.Compress(body, compressMethod, compressMethod)
if err != nil {
return nil, err
}
} else {
compressBytes = body
}
return compressBytes, nil
}
go-utils/gojson/gojson.go (1.4 KiB)
package gojson
import (
"bytes"
"compress/gzip"
"encoding/json"
"errors"
"io"
"os"
jsoniter "github.com/json-iterator/go"
"github.com/oliveagle/jsonpath"
)
// 在项目入口统一配置
var JSON = jsoniter.ConfigCompatibleWithStandardLibrary
func Pretty(strJson string) string {
var out bytes.Buffer
if err := json.Indent(&out, ([]byte)(strJson), "", " "); err != nil {
return ""
}
return out.String()
}
func Marshal(obj any, pretty bool) string {
var data []byte
if pretty {
data, _ = JSON.MarshalIndent(obj, "", " ")
} else {
data, _ = JSON.Marshal(obj)
}
return string(data)
}
func UnmarshalFromFile(filePath string, val any) error {
data, err := os.ReadFile(filePath)
if err != nil {
return err
}
return JSON.Unmarshal(data, val)
}
func Unmarshal(data string, val any) error {
return JSON.UnmarshalFromString(data, val)
}
func UnmarshalFromGzip(data []byte, val any) error {
reader, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return err
}
defer reader.Close()
orginData, err := io.ReadAll(reader)
if err != nil {
return err
}
return JSON.Unmarshal(orginData, val)
}
func PathLookup[T any](obj interface{}, jpath string) (T, error) {
var tval T
var err error
var ok bool
val, err := jsonpath.JsonPathLookup(obj, jpath)
if err != nil {
return tval, err
}
tval, ok = val.(T)
if !ok {
return tval, errors.New("type conversion failed")
}
return tval, nil
}
go-utils/gojson/readme.md (221 B)
A high-performance 100% compatible drop-in replacement of "encoding/json"
高性能json解析
- 基于 github.com/json-iterator/go
- 提供 PathLookup辅助方法 支持jsonPath(github.com/oliveagle/jsonpath)
go-utils/goprometheus/client.go (1.1 KiB)
package goprometheus
import (
"errors"
golog "github.com/gif-gif/go.io/go-log"
)
var __clients = map[string]*GoPrometheus{}
// 可以一次初始化多个Redis实例或者 多次调用初始化多个实例
func Init(configs ...Config) (err error) {
for _, conf := range configs {
name := conf.Name
if name == "" {
name = "default"
}
if __clients[name] != nil {
return errors.New("GoPrometheus client [" + name + "] already exists")
}
__clients[name], err = New(conf)
if err != nil {
return err
}
}
return
}
func GetClient(names ...string) *GoPrometheus {
name := "default"
if l := len(names); l > 0 {
name = names[0]
if name == "" {
name = "default"
}
}
if cli, ok := __clients[name]; ok {
return cli
}
return nil
}
func DelClient(names ...string) {
if l := len(names); l > 0 {
for _, name := range names {
delete(__clients, name)
}
}
}
func Default() *GoPrometheus {
if cli, ok := __clients["default"]; ok {
return cli
}
if l := len(__clients); l == 1 {
for _, cli := range __clients {
return cli
}
}
golog.WithTag("GoPrometheus").Error("no default prometheus client")
return nil
}
go-utils/goprometheus/config.go (397 B)
package goprometheus
type Config struct {
Name string `json:"name,optional" yaml:"Name"`
PrometheusUrl string `json:"prometheusUrl,optional" yaml:"PrometheusUrl" default:"0.0.0.0:9090"`
Filters []string `json:"filters,optional" yaml:"Filters"`
}
func (c *Config) GetUrl() string {
if c.PrometheusUrl == "" {
c.PrometheusUrl = "0.0.0.0:9090"
}
return c.PrometheusUrl
}
go-utils/goprometheus/consts.go (5.1 KiB)
package goprometheus
import (
"github.com/prometheus/common/model"
)
var (
MetricVersion = "version"
MetricStartupTimestamp = "startup_timestamp"
MetricAliveTimestamp = "alive_timestamp"
MetricConfigUpdateTimestamp = "config_update_timestamp"
MetricRunTime = "run_time"
MetricErrCode = "err_code"
MetricTotalBandwidthIn = "total_bandwidth_in"
MetricTotalBandwidthOut = "total_bandwidth_out"
MetricMemberLevelUserCount = "member_level_user_count"
MetricRealMemberLevelUserCount = "real_member_level_user_count"
MetricIpTcpConnectionsIn = "ip_tcp_connections_in"
MetricIpTcpConnectionsOut = "ip_tcp_connections_out"
MetricIpUdpConnectionsIn = "ip_udp_connections_in"
MetricIpUdpPortsOut = "ip_udp_ports_out"
MetricIpBandwidthIn = "ip_bandwidth_in"
MetricIpBandwidthOut = "ip_bandwidth_out"
MetricIpUserCount = "ip_user_count"
MetricNodeExporterBuildInfo = "node_exporter_build_info"
MetricNodeCpuSecondsTotal = "node_cpu_seconds_total"
MetricNodeCpuGuessSecondsTotal = "node_cpu_guess_seconds_total"
MetricNodeMemTotal = "node_memory_MemTotal_bytes"
MetricNodeMemAvailable = "node_memory_MemAvailable_bytes"
MetricNodeDiskSize = "node_filesystem_size_bytes"
MetricNodeDiskFree = "node_filesystem_free_bytes"
MetricNodeDiskAvailable = "node_filesystem_avail_bytes"
MetricNodeDiskInode = "node_filesystem_files"
MetricNodeDiskInodeFree = "node_filesystem_files_free"
MetricNodeDiskReadonly = "node_filesystem_readonly"
MetricNodeDiskDeviceError = "node_filesystem_device_error"
MetricNodeTrafficIn = "node_network_receive_bytes_total"
MetricNodeTrafficOut = "node_network_transmit_bytes_total"
MetricUp = "up"
// node_memory_Active_anon_bytes
// node_memory_Active_bytes
// node_memory_Active_file_bytes
// node_memory_AnonHugePages_bytes
// node_memory_AnonPages_bytes
// node_memory_Bounce_bytes
// node_memory_Buffers_bytes
// node_memory_Cached_bytes
// node_memory_CommitLimit_bytes
// node_memory_Committed_AS_bytes
// node_memory_DirectMap1G_bytes
// node_memory_DirectMap2M_bytes
// node_memory_DirectMap4k_bytes
// node_memory_Dirty_bytes
// node_memory_FileHugePages_bytes
// node_memory_FilePmdMapped_bytes
// node_memory_HardwareCorrupted_bytes
// node_memory_HugePages_Free
// node_memory_HugePages_Rsvd
// node_memory_HugePages_Surp
// node_memory_HugePages_Total
// node_memory_Hugepagesize_bytes
// node_memory_Hugetlb_bytes
// node_memory_Inactive_anon_bytes
// node_memory_Inactive_bytes
// node_memory_Inactive_file_bytes
// node_memory_KReclaimable_bytes
// node_memory_KernelStack_bytes
// node_memory_Mapped_bytes
// node_memory_MemAvailable_bytes
// node_memory_MemFree_bytes
// node_memory_MemTotal_bytes
// node_memory_Mlocked_bytes
// node_memory_NFS_Unstable_bytes
// node_memory_PageTables_bytes
// node_memory_Percpu_bytes
// node_memory_SReclaimable_bytes
// node_memory_SUnreclaim_bytes
// node_memory_ShmemHugePages_bytes
// node_memory_ShmemPmdMapped_bytes
// node_memory_Shmem_bytes
// node_memory_Slab_bytes
// node_memory_SwapCached_bytes
// node_memory_SwapFree_bytes
// node_memory_SwapTotal_bytes
// node_memory_Unevictable_bytes
// node_memory_VmallocChunk_bytes
// node_memory_VmallocTotal_bytes
// node_memory_VmallocUsed_bytes
// node_memory_WritebackTmp_bytes
// node_memory_Writeback_bytes
// node_network_receive_bytes_total
// node_network_receive_compressed_total
// node_network_receive_drop_total
// node_network_receive_errs_total
// node_network_receive_fifo_total
// node_network_receive_frame_total
// node_network_receive_multicast_total
// node_network_receive_nohandler_total
// node_network_receive_packets_total
// node_network_transmit_bytes_total
// node_network_transmit_carrier_total
// node_network_transmit_colls_total
// node_network_transmit_compressed_total
// node_network_transmit_drop_total
// node_network_transmit_errs_total
// node_network_transmit_fifo_total
// node_network_transmit_packets_total
// node_scrape_collector_duration_seconds
// node_scrape_collector_success
)
var (
MetricLabelName = model.LabelName("__name__")
MetricLabelServerType = model.LabelName("server_type")
MetricLabelJob = model.LabelName("job")
MetricLabelGroup = model.LabelName("group")
MetricLabelProductIds = model.LabelName("product_ids")
MetricLabelSupplierId = model.LabelName("supplier_id")
MetricLabelRetry = model.LabelName("retry")
MetricLabelInstanceId = model.LabelName("instance_id")
MetricLabelCpu = model.LabelName("cpu")
MetricLabelExportedInstanceId = model.LabelName("exported_instance_id")
MetricLabelInstance = model.LabelName("instance")
MetricLabelMemberLevel = model.LabelName("member_level")
MetricLabelIp = model.LabelName("ip")
MetricLabelProtocol = model.LabelName("protocol")
MetricLabelPort = model.LabelName("port")
)
go-utils/goprometheus/goprometheusx/agent.go (1.0 KiB)
package goprometheusx
import (
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus/promhttp"
"net/http"
"sync"
)
var (
once sync.Once
)
// StartAgent starts a prometheus agent.
func StartAgent(addr, path string) {
if len(addr) < 3 {
//端口可能冲突
//addr = "0.0.0.0:9101"
return
}
if path == "" {
path = "/metrics"
}
once.Do(func() {
goutils.AsyncFunc(func() {
http.Handle(path, promhttp.Handler())
golog.InfoF("Starting prometheus agent at %s%s", addr, path)
if err := http.ListenAndServe(addr, nil); err != nil {
golog.Error(err)
}
//endless.NewServer(addr, promhttp.Handler()).ListenAndServe()
})
})
}
func PromHandler(handler http.Handler) gin.HandlerFunc {
return func(c *gin.Context) {
handler.ServeHTTP(c.Writer, c.Request)
}
}
func PrometheusBind(engine *gin.Engine, path string) {
if path == "" {
path = "/metrics"
}
engine.GET(path, PromHandler(promhttp.Handler()))
}
go-utils/goprometheus/goprometheusx/config.go (202 B)
package goprometheusx
type Config struct {
Host string `json:"host,optional" yaml:"host"`
Port int `json:"port,default=9101" yaml:"port"`
Path string `json:"path,default=/metrics" yaml:"path"`
}
go-utils/goprometheus/goprometheusx/counter.go (1.1 KiB)
package goprometheusx
import (
prom "github.com/prometheus/client_golang/prometheus"
)
type (
// A CounterVecOpts is an alias of VectorOpts.
CounterVecOpts VectorOpts
// CounterVec interface represents a counter vector.
CounterVec interface {
// Inc increments labels.
Inc(labels ...string)
// Add adds labels with v.
Add(v float64, labels ...string)
close() bool
}
promCounterVec struct {
counter *prom.CounterVec
}
)
// NewCounterVec returns a CounterVec.
func NewCounterVec(cfg *CounterVecOpts) CounterVec {
if cfg == nil {
return nil
}
vec := prom.NewCounterVec(prom.CounterOpts{
Namespace: cfg.Namespace,
Subsystem: cfg.Subsystem,
Name: cfg.Name,
Help: cfg.Help,
}, cfg.Labels)
prom.MustRegister(vec)
cv := &promCounterVec{
counter: vec,
}
return cv
}
func (cv *promCounterVec) Inc(labels ...string) {
cv.counter.WithLabelValues(labels...).Inc()
}
func (cv *promCounterVec) Add(v float64, labels ...string) {
cv.counter.WithLabelValues(labels...).Add(v)
}
func (cv *promCounterVec) close() bool {
return prom.Unregister(cv.counter)
}
go-utils/goprometheus/goprometheusx/gauge.go (1.2 KiB)
package goprometheusx
import (
prom "github.com/prometheus/client_golang/prometheus"
)
type (
// GaugeVecOpts is an alias of VectorOpts.
GaugeVecOpts VectorOpts
// GaugeVec represents a gauge vector.
GaugeVec interface {
// Set sets v to labels.
Set(v float64, labels ...string)
// Inc increments labels.
Inc(labels ...string)
// Add adds v to labels.
Add(v float64, labels ...string)
close() bool
}
promGaugeVec struct {
gauge *prom.GaugeVec
}
)
// NewGaugeVec returns a GaugeVec.
func NewGaugeVec(cfg *GaugeVecOpts) GaugeVec {
if cfg == nil {
return nil
}
vec := prom.NewGaugeVec(
prom.GaugeOpts{
Namespace: cfg.Namespace,
Subsystem: cfg.Subsystem,
Name: cfg.Name,
Help: cfg.Help,
}, cfg.Labels)
prom.MustRegister(vec)
gv := &promGaugeVec{
gauge: vec,
}
return gv
}
func (gv *promGaugeVec) Inc(labels ...string) {
gv.gauge.WithLabelValues(labels...).Inc()
}
func (gv *promGaugeVec) Add(v float64, labels ...string) {
gv.gauge.WithLabelValues(labels...).Add(v)
}
func (gv *promGaugeVec) Set(v float64, labels ...string) {
gv.gauge.WithLabelValues(labels...).Set(v)
}
func (gv *promGaugeVec) close() bool {
return prom.Unregister(gv.gauge)
}
go-utils/goprometheus/goprometheusx/histogram.go (1.1 KiB)
package goprometheusx
import (
prom "github.com/prometheus/client_golang/prometheus"
)
type (
// A HistogramVecOpts is a histogram vector options.
HistogramVecOpts struct {
Namespace string
Subsystem string
Name string
Help string
Labels []string
Buckets []float64
}
// A HistogramVec interface represents a histogram vector.
HistogramVec interface {
// Observe adds observation v to labels.
Observe(v int64, labels ...string)
close() bool
}
promHistogramVec struct {
histogram *prom.HistogramVec
}
)
// NewHistogramVec returns a HistogramVec.
func NewHistogramVec(cfg *HistogramVecOpts) HistogramVec {
if cfg == nil {
return nil
}
vec := prom.NewHistogramVec(prom.HistogramOpts{
Namespace: cfg.Namespace,
Subsystem: cfg.Subsystem,
Name: cfg.Name,
Help: cfg.Help,
Buckets: cfg.Buckets,
}, cfg.Labels)
prom.MustRegister(vec)
hv := &promHistogramVec{
histogram: vec,
}
return hv
}
func (hv *promHistogramVec) Observe(v int64, labels ...string) {
hv.histogram.WithLabelValues(labels...).Observe(float64(v))
}
func (hv *promHistogramVec) close() bool {
return prom.Unregister(hv.histogram)
}
go-utils/goprometheus/goprometheusx/metric.go (186 B)
package goprometheusx
// A VectorOpts is a general configuration.
type VectorOpts struct {
Namespace string
Subsystem string
Name string
Help string
Labels []string
}
go-utils/goprometheus/goprometheusx/prometheus.go (1.2 KiB)
package goprometheusx
import (
"fmt"
"github.com/patrickmn/go-cache"
"github.com/zeromicro/go-zero/core/metric"
"time"
)
const (
LevelWarning = "warn"
LevelError = "error"
LevelPanic = "panic"
)
var (
metricServerErrorTotal = metric.NewCounterVec(&metric.CounterVecOpts{
Namespace: "server",
Subsystem: "alert",
Name: "alert_count",
Help: "server error",
Labels: []string{"level", "module", "name"},
})
sharedCache = cache.New(time.Minute, 5*time.Minute)
)
func Init(config Config) {
addr := fmt.Sprintf("%s:%d", config.Host, config.Port)
StartAgent(addr, config.Path)
}
func Alert(level, module, name string) error {
key := level + module + name
if _, ok := sharedCache.Get(key); ok {
return fmt.Errorf("alert too fast:%s", key)
}
if len(name) > 50 {
return fmt.Errorf("name is too long,max is 50, %s", name)
}
metricServerErrorTotal.Inc(level, module, name)
sharedCache.Set(key, struct{}{}, time.Second*2)
return nil
}
func AlertErr(module, name string) {
Alert(LevelError, module, name)
}
func AlertWarn(module, name string) {
Alert(LevelWarning, module, name)
}
func AlertPanic(module, name string) {
Alert(LevelPanic, module, name)
}
go-utils/goprometheus/prometheus.go (891 B)
package goprometheus
import (
prometheusApi "github.com/prometheus/client_golang/api"
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
)
type GoPrometheus struct {
Client prometheusApi.Client
Api v1.API
Filters []string // 服务器过滤条件
}
func New(config Config) (*GoPrometheus, error) {
client, err := prometheusApi.NewClient(prometheusApi.Config{
Address: config.GetUrl(),
})
if err != nil {
return nil, err
}
v1API := v1.NewAPI(client)
return &GoPrometheus{
Client: client,
Api: v1API,
Filters: config.Filters,
}, nil
}
func (g *GoPrometheus) AddFilters(filters ...string) {
g.Filters = append(g.Filters, filters...)
}
func (g *GoPrometheus) GetFilters() []string {
return g.Filters
}
func (g *GoPrometheus) SetFilters(filters ...string) {
if len(g.Filters) == 0 {
g.Filters = []string{}
} else {
g.Filters = filters
}
}
go-utils/goprometheus/query.go (1.4 KiB)
package goprometheus
import (
"context"
"fmt"
"strings"
"time"
"github.com/prometheus/common/model"
"github.com/zeromicro/go-zero/core/logx"
)
func (g *GoPrometheus) PrometheusQuery(ctx context.Context, query string) (model.Vector, error) {
logx.WithContext(ctx).Debugf("[prometheusQuery] executing query: %s", query)
result, warnings, err := g.Api.Query(ctx, query, time.Now())
if err != nil {
logx.WithContext(ctx).Errorf("[prometheusQuery] prometheus query failed: %v", err)
return nil, err
}
if len(warnings) > 0 {
logx.WithContext(ctx).Infof("[prometheusQuery] prometheus query warnings: %s, warnings: %v", query, warnings)
}
vector, ok := result.(model.Vector)
if !ok {
err := fmt.Errorf("prometheus query result type unexpected: %v", result)
logx.WithContext(ctx).Errorf("[prometheusQuery] %v", err)
return nil, err
}
return vector, nil
}
// filters = append(filters, g.GetFilters()...) //把内部filters 加上
// queryStr := fmt.Sprintf(`%s{%s}`, metrics, strings.Join(filters, ","))
//
// 执行实例 real_member_level_user_count{job="fkey-node",group="all",instance_id=~"2015"}
func (g *GoPrometheus) PrometheusQueryMetrics(ctx context.Context, metrics string, filters []string) (model.Vector, error) {
filters = append(filters, g.GetFilters()...) //把内部filters 加上
queryStr := fmt.Sprintf(`%s{%s}`, metrics, strings.Join(filters, ","))
return g.PrometheusQuery(ctx, queryStr)
}
go-utils/goprometheus/readme.md (1.0 KiB)
Prometheus 系统指标
- [x] 基于 https://github.com/prometheus/node_exporter
一、 Labels 格式 job=%s-node-exporter
二、 查询系统指标过滤表达式实例,默认以产品分类
fmt.Sprintf(`%s="%s-node-exporter"`, MetricLabelJob, query.ProductCode)
三、用法
package main
import (
"context"
"fmt"
"github.com/gif-gif/go.io/go-utils/goprometheus"
)
func main() {
product := "fkey"
err := goprometheus.Init(goprometheus.Config{
Name: product,
PrometheusUrl: "http://127.0.0.1:9091",
Filters: []string{
//fmt.Sprintf(`%s="%s-node-exporter"`, goprometheus.MetricLabelJob, product),
fmt.Sprintf(`%s="%s-node"`, goprometheus.MetricLabelJob, product),
},
})
if err != nil {
panic(err)
}
query := goprometheus.MetricQuery{
ProductCode: product,
Group: "all",
InstanceIds: []int64{2015},
}
memberCount, err := goprometheus.GetClient(product).SvrRealMemberLevelUserCount(context.Background(), query)
if err != nil {
panic(err)
}
fmt.Println(memberCount)
}
go-utils/goprometheus/svr_member_level_user_count.go (1.9 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"github.com/prometheus/common/model"
"strings"
)
func (g *GoPrometheus) GetSvrMemberLevelUserCount(ctx context.Context, metrics string, query MetricQuery) (model.Vector, error) {
filters := []string{}
filters = append(filters, g.Filters...)
filters = ToGroupFilter(filters, query.Group)
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
queryStr := fmt.Sprintf(`%s{%s}`, metrics, strings.Join(filters, ","))
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSvrMemberLevelUserCount(vector *model.Vector, result map[int64][]*MemberLevelUserCount) map[int64][]*MemberLevelUserCount {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
memberLevel := gconv.Int(string(sample.Metric[MetricLabelMemberLevel]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = []*MemberLevelUserCount{}
}
result[instanceId] = append(result[instanceId], &MemberLevelUserCount{
Level: int64(memberLevel),
Count: int64(sample.Value),
})
}
return result
}
// 获取每个会员等级的用户数(包含转发)
func (g *GoPrometheus) SvrMemberLevelUserCount(ctx context.Context, query MetricQuery) (map[int64][]*MemberLevelUserCount, error) {
vector, err := g.GetSvrMemberLevelUserCount(ctx, MetricMemberLevelUserCount, query)
if err != nil {
return nil, err
}
return g.PreHandleSvrMemberLevelUserCount(&vector, map[int64][]*MemberLevelUserCount{}), nil
}
// 获取每个会员等级的用户数(不包含转发)
func (g *GoPrometheus) SvrRealMemberLevelUserCount(ctx context.Context, query MetricQuery) (map[int64][]*MemberLevelUserCount, error) {
vector, err := g.GetSvrMemberLevelUserCount(ctx, MetricRealMemberLevelUserCount, query)
if err != nil {
return nil, err
}
return g.PreHandleSvrMemberLevelUserCount(&vector, map[int64][]*MemberLevelUserCount{}), nil
}
go-utils/goprometheus/sys_bandwidth_in.go (1.6 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取各个服务器的总入站带宽 单位 byte/s, 如果不指定时间窗口,则默认获取90s内的数据来计算
func (g *GoPrometheus) GetSysBandwidthIn(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`device!~"tap.*|veth.*|br.*|docker.*|virbr*|lo*|cni.*|ifb.*"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "90s"
}
queryStr := fmt.Sprintf(`max by (%s)(rate(%s{%s}[%s]))`, MetricLabelInstanceId, MetricNodeTrafficIn, strings.Join(filters, ","), timeRange)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysBandwidthIn(vector *model.Vector, result map[int64]*Bandwidth) map[int64]*Bandwidth {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &Bandwidth{}
}
result[instanceId].In = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysBandwidthIn(ctx context.Context, query MetricQuery) (map[int64]*Bandwidth, error) {
vector, err := g.GetSysBandwidthIn(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*Bandwidth)
result = g.PreHandleSysBandwidthIn(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_bandwidth_out.go (1.6 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取各个服务器的总出站带宽 单位 byte/s, 如果不指定时间窗口,则默认获取90s内的数据来计算
func (g *GoPrometheus) GetSysBandwidthOut(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`device!~"tap.*|veth.*|br.*|docker.*|virbr*|lo*|cni.*|ifb.*"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "90s"
}
queryStr := fmt.Sprintf(`max by (%s) (rate(%s{%s}[%s]))`, MetricLabelInstanceId, MetricNodeTrafficOut, strings.Join(filters, ","), timeRange)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysBandwidthOut(vector *model.Vector, result map[int64]*Bandwidth) map[int64]*Bandwidth {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &Bandwidth{}
}
result[instanceId].Out = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysBandwidthOut(ctx context.Context, query MetricQuery) (map[int64]*Bandwidth, error) {
vector, err := g.GetSysBandwidthOut(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*Bandwidth)
result = g.PreHandleSysBandwidthOut(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_cpu_core_count.go (1.5 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取服务器cpu核心数(不是物理核心数,而是逻辑核心数。开启超线程后,一个物理核心可以对应多个逻辑核心,当前的指标中无法得知是否开启超线程,也无法区分物理核心和逻辑核心)
func (g *GoPrometheus) GetSysCpuCoreCount(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
queryStr := fmt.Sprintf(`count by (%s) (count by (%s,%s) (%s{%s}))`, MetricLabelInstanceId, MetricLabelInstanceId, MetricLabelCpu, MetricNodeCpuSecondsTotal, strings.Join(filters, ","))
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysCpuCoreCount(vector *model.Vector, result map[int64]int64) map[int64]int64 {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
result[instanceId] = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysCpuCoreCount(ctx context.Context, query MetricQuery) (map[int64]int64, error) {
vector, err := g.GetSysCpuCoreCount(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]int64)
result = g.PreHandleSysCpuCoreCount(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_cpu_usage_rate.go (1.5 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取服务器cpu使用率 单位 %, 如果不指定时间窗口,则默认获取5分钟内的数据来计算
func (g *GoPrometheus) GetSysCpuUsageRate(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`mode="idle"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "5m"
}
queryStr := fmt.Sprintf(`(1 - avg(rate(%s{%s}[%s])) by (%s)) * 100`, MetricNodeCpuSecondsTotal, strings.Join(filters, ","), timeRange, MetricLabelInstanceId)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysCpuUsage(vector *model.Vector, result map[int64]*SysUsage) map[int64]*SysUsage {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &SysUsage{}
}
result[instanceId].CpuUsage = float64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysCpuUsageRate(ctx context.Context, query MetricQuery) (model.Vector, error) {
vector, err := g.GetSysCpuUsageRate(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*SysUsage)
result = g.PreHandleSysCpuUsage(&vector, result)
return vector, nil
}
go-utils/goprometheus/sys_disk_total.go (1.3 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取服务器磁盘总大小 单位 byte
func (g *GoPrometheus) GetSysDiskTotal(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`fstype=~"ext.?|xfs"`,
`mountpoint="/"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
queryStr := fmt.Sprintf(`%s{%s}`, MetricNodeDiskSize, strings.Join(filters, ","))
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysDiskTotal(vector *model.Vector, result map[int64]*SysUsage) map[int64]*SysUsage {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &SysUsage{}
}
result[instanceId].DiskTotal = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysDiskTotal(ctx context.Context, query MetricQuery) (map[int64]*SysUsage, error) {
vector, err := g.GetSysDiskTotal(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*SysUsage)
result = g.PreHandleSysDiskTotal(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_disk_usage.go (1.6 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取服务器磁盘使用率 单位 %
func (g *GoPrometheus) GetSysDiskUsage(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`fstype=~"ext.?|xfs"`,
`mountpoint="/"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
totalQuery := fmt.Sprintf(`%s{%s}`, MetricNodeDiskSize, strings.Join(filters, ","))
freeQuery := fmt.Sprintf(`%s{%s}`, MetricNodeDiskFree, strings.Join(filters, ","))
availableQuery := fmt.Sprintf(`%s{%s}`, MetricNodeDiskAvailable, strings.Join(filters, ","))
queryStr := fmt.Sprintf(`max by(%s) ((%s-%s)*100/(%s+(%s-%s)))`, MetricLabelInstanceId, totalQuery, freeQuery, availableQuery, totalQuery, freeQuery)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysDiskUsage(vector *model.Vector, result map[int64]*SysUsage) map[int64]*SysUsage {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &SysUsage{}
}
result[instanceId].DiskUsage = float64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysDiskUsage(ctx context.Context, query MetricQuery) (map[int64]*SysUsage, error) {
vector, err := g.GetSysDiskUsage(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*SysUsage)
result = g.PreHandleSysDiskUsage(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_memory_total.go (1.3 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取服务器内存总大小 单位 byte
func (g *GoPrometheus) GetSysMemoryTotal(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
queryStr := fmt.Sprintf(`%s{%s}`, MetricNodeMemTotal, strings.Join(filters, ","))
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysMemoryTotal(vector *model.Vector, result map[int64]*SysUsage) map[int64]*SysUsage {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &SysUsage{}
}
result[instanceId].MemoryTotal = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysMemoryTotal(ctx context.Context, query MetricQuery) (map[int64]*SysUsage, error) {
vector, err := g.GetSysMemoryTotal(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*SysUsage)
result = g.PreHandleSysMemoryTotal(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_memory_usage.go (1.5 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取服务器内存使用率 单位 %
func (g *GoPrometheus) GetSysMemoryUsage(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
totalQuery := fmt.Sprintf(`%s{%s}`, MetricNodeMemTotal, strings.Join(filters, ","))
availableQuery := fmt.Sprintf(`%s{%s}`, MetricNodeMemAvailable, strings.Join(filters, ","))
queryStr := fmt.Sprintf(`(1 - (%s / %s)) * 100`, availableQuery, totalQuery)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysMemoryUsage(vector *model.Vector, result map[int64]*SysUsage) map[int64]*SysUsage {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &SysUsage{}
}
result[instanceId].MemoryUsage = float64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysMemoryUsage(ctx context.Context, query MetricQuery) (map[int64]*SysUsage, error) {
vector, err := g.GetSysMemoryUsage(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*SysUsage)
result = g.PreHandleSysMemoryUsage(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_total_bandwidth_in.go (1.6 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取所有服务器的网卡总入站带宽 单位 byte/s, 如果不指定时间窗口,则默认获取90s内的数据来计算
func (g *GoPrometheus) GetSysTotalBandwidthIn(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`device!~"tap.*|veth.*|br.*|docker.*|virbr*|lo*|cni.*|ifb.*"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "90s"
}
queryStr := fmt.Sprintf(`sum(rate(%s{%s}[%s]))`, MetricNodeTrafficIn, strings.Join(filters, ","), timeRange)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysTotalBandwidthIn(vector *model.Vector, result map[int64]*Bandwidth) map[int64]*Bandwidth {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &Bandwidth{}
}
result[instanceId].In = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysTotalBandwidthIn(ctx context.Context, query MetricQuery) (map[int64]*Bandwidth, error) {
vector, err := g.GetSysTotalBandwidthIn(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*Bandwidth)
result = g.PreHandleSysTotalBandwidthIn(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_total_bandwidth_out.go (1.6 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取所有服务器的网卡总出站带宽 单位 byte/s, 如果不指定时间窗口,则默认获取90s内的数据来计算
func (g *GoPrometheus) GetSysTotalBandwidthOut(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`device!~"tap.*|veth.*|br.*|docker.*|virbr*|lo*|cni.*|ifb.*"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "90s"
}
queryStr := fmt.Sprintf(`sum(rate(%s{%s}[%s]))`, MetricNodeTrafficOut, strings.Join(filters, ","), timeRange)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysTotalBandwidthOut(vector *model.Vector, result map[int64]*Bandwidth) map[int64]*Bandwidth {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &Bandwidth{}
}
result[instanceId].Out = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysTotalBandwidthOut(ctx context.Context, query MetricQuery) (map[int64]*Bandwidth, error) {
vector, err := g.GetSysTotalBandwidthOut(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*Bandwidth)
result = g.PreHandleSysTotalBandwidthOut(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_total_traffic_in.go (1.5 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取所有服务器的网卡总入站流量 单位 byte, 如果不指定时间窗口,则默认获取10分钟内的数据来计算
func (g *GoPrometheus) GetSysTotalTrafficIn(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`device!~"tap.*|veth.*|br.*|docker.*|virbr*|lo*|cni.*|ifb.*"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "10m"
}
queryStr := fmt.Sprintf(`sum(increase(%s{%s}[%s]))`, MetricNodeTrafficIn, strings.Join(filters, ","), timeRange)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysTotalTrafficIn(vector *model.Vector, result map[int64]*Traffic) map[int64]*Traffic {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &Traffic{}
}
result[instanceId].In = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysTotalTrafficIn(ctx context.Context, query MetricQuery) (map[int64]*Traffic, error) {
vector, err := g.GetSysTotalTrafficIn(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*Traffic)
result = g.PreHandleSysTotalTrafficIn(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_total_traffic_out.go (1.6 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取所有服务器的网卡总出站流量 单位 byte, 如果不指定时间窗口,则默认获取10分钟内的数据来计算
func (g *GoPrometheus) GetSysTotalTrafficOut(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`device!~"tap.*|veth.*|br.*|docker.*|virbr*|lo*|cni.*|ifb.*"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "10m"
}
queryStr := fmt.Sprintf(`sum(increase(%s{%s}[%s]))`, MetricNodeTrafficOut, strings.Join(filters, ","), timeRange)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysTotalTrafficOut(vector *model.Vector, result map[int64]*Traffic) map[int64]*Traffic {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &Traffic{}
}
result[instanceId].Out = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysTotalTrafficOut(ctx context.Context, query MetricQuery) (map[int64]*Traffic, error) {
vector, err := g.GetSysTotalTrafficOut(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*Traffic)
result = g.PreHandleSysTotalTrafficOut(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_traffic_in.go (1.6 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取各个服务器的网卡总入站流量 单位 byte, 如果不指定时间窗口,则默认获取10分钟内的数据来计算
func (g *GoPrometheus) GetSysTrafficIn(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`device!~"tap.*|veth.*|br.*|docker.*|virbr*|lo*|cni.*|ifb.*"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "10m"
}
queryStr := fmt.Sprintf(`sum by (%s) (increase(%s{%s}[%s]))`, MetricLabelInstanceId, MetricNodeTrafficIn, strings.Join(filters, ","), timeRange)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysTrafficIn(vector *model.Vector, result map[int64]*Traffic) map[int64]*Traffic {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &Traffic{}
}
result[instanceId].In = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysTrafficIn(ctx context.Context, query MetricQuery) (map[int64]*Traffic, error) {
vector, err := g.GetSysTrafficIn(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*Traffic)
result = g.PreHandleSysTrafficIn(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_traffic_out.go (1.6 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取各个服务器的网卡总出站流量 单位 byte, 如果不指定时间窗口,则默认获取10分钟内的数据来计算
func (g *GoPrometheus) GetSysTrafficOut(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{
`device!~"tap.*|veth.*|br.*|docker.*|virbr*|lo*|cni.*|ifb.*"`,
}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
timeRange := query.TimeRange
if timeRange == "" {
timeRange = "10m"
}
queryStr := fmt.Sprintf(`sum by (%s) (increase(%s{%s}[%s]))`, MetricLabelInstanceId, MetricNodeTrafficOut, strings.Join(filters, ","), timeRange)
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysTrafficOut(vector *model.Vector, result map[int64]*Traffic) map[int64]*Traffic {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
if _, ok := result[instanceId]; !ok {
result[instanceId] = &Traffic{}
}
result[instanceId].Out = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysTrafficOut(ctx context.Context, query MetricQuery) (map[int64]*Traffic, error) {
vector, err := g.GetSysTrafficOut(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]*Traffic)
result = g.PreHandleSysTrafficOut(&vector, result)
return result, nil
}
go-utils/goprometheus/sys_up.go (1.1 KiB)
package goprometheus
import (
"context"
"fmt"
"github.com/gogf/gf/util/gconv"
"strings"
"github.com/prometheus/common/model"
)
// 获取服务器在线状态
func (g *GoPrometheus) GetSysUp(ctx context.Context, query MetricQuery) (model.Vector, error) {
filters := []string{}
filters = append(filters, g.Filters...)
// filters = toGroupFilter(filters, query.Group) // node-exporter 没有 group 标签
filters = ToInstanceIdsFilter(filters, query.InstanceIds)
queryStr := fmt.Sprintf(`%s{%s}`, MetricUp, strings.Join(filters, ","))
return g.PrometheusQuery(ctx, queryStr)
}
func (g *GoPrometheus) PreHandleSysUp(vector *model.Vector, result map[int64]int64) map[int64]int64 {
for _, sample := range *vector {
instanceId := gconv.Int64(string(sample.Metric[MetricLabelInstanceId]))
result[instanceId] = int64(sample.Value)
}
return result
}
func (g *GoPrometheus) SysUp(ctx context.Context, query MetricQuery) (map[int64]int64, error) {
vector, err := g.GetSysUp(ctx, query)
if err != nil {
return nil, err
}
result := make(map[int64]int64)
result = g.PreHandleSysUp(&vector, result)
return result, nil
}
go-utils/goprometheus/types.go (2.0 KiB)
package goprometheus
import "github.com/prometheus/common/model"
type MetricName = string
type MetricLabel = model.LabelName
type MetricQuery = struct {
ProductCode string
Group string
InstanceIds []int64
TimeRange string
}
type Bandwidth struct {
In int64 `json:"in,optional"`
Out int64 `json:"out,optional"`
}
type SysUsage struct {
CpuUsage float64 `json:"cpuUsage,optional"` // CPU使用率
DiskTotal int64 `json:"diskTotal,optional"` // 磁盘总大小
DiskUsage float64 `json:"diskUsage,optional"` // 磁盘使用率
MemoryTotal int64 `json:"memoryTotal,optional"` // 内存总大小
MemoryUsage float64 `json:"memoryUsage,optional"` // 内存使用率
}
type Traffic struct {
In int64 `json:"in,optional"` // 入站流量
Out int64 `json:"out,optional"` // 出站流量
Total int64 `json:"total,optional"` // 总流量
}
// 会员等级用户数
type MemberLevelUserCount struct {
Level int64 `json:"level,optional"` // 会员等级
Count int64 `json:"count,optional"` // 用户数
}
type IpUserCount struct {
Ip string `json:"ip,optional"` // IP地址
Count int64 `json:"count,optional"` // 用户数
}
type IpBandwidth struct {
Ip string `json:"ip,optional"` // IP地址
Details []*ProtocolBandwidth `json:"details,optional,omitempty"` // 带宽详情
}
type IpConnCount struct {
Ip string `json:"ip,optional"` // IP地址
Details []*ProtocolConnCount `json:"details,optional,omitempty"` // 连接数详情
}
type ProtocolBandwidth struct {
Protocol string `json:"protocol,optional"` // 协议
Port int64 `json:"port,optional"` // 端口
In int64 `json:"in,optional"` // 入站带宽
Out int64 `json:"out,optional"` // 出站带宽
}
type ProtocolConnCount struct {
Protocol string `json:"protocol,optional"` // 协议
Port int64 `json:"port,optional"` // 端口
In int64 `json:"in,optional"` // 入站连接数
Out int64 `json:"out,optional"` // 出站连接数
}
go-utils/goprometheus/utils.go (833 B)
package goprometheus
import (
"fmt"
"strconv"
"strings"
)
func ToGroupFilter(filters []string, group string) []string {
if group == "" {
return filters
} else {
return ToFilter(filters, string(MetricLabelGroup), group)
}
}
// `%s=~"%s"` 包含关系
func ToInstanceIdsFilter(filters []string, instanceIds []int64) []string {
if len(instanceIds) == 0 {
return filters
}
strIds := make([]string, len(instanceIds))
for i, id := range instanceIds {
strIds[i] = strconv.Itoa(int(id))
}
return append(filters, fmt.Sprintf(`%s=~"%s"`, MetricLabelInstanceId, strings.Join(strIds, "|")))
}
// 通用精确过滤器(`%s="%s"`)
func ToFilter(filters []string, metrics string, value string) []string {
if value == "" {
return filters
} else {
return append(filters, fmt.Sprintf(`%s="%s"`, metrics, value))
}
}
go-utils/gorandom/gorandom.go (5.7 KiB)
package gorandom
import (
"fmt"
"math/rand/v2"
"sync"
"time"
)
// RandomGenerator 结构体定义 ,线程安全
type RandomGenerator struct {
mu sync.Mutex
rng *rand.Rand
}
// 创建新的随机数生成器
func NewRandomGenerator() *RandomGenerator {
// Go 1.20+ 自动使用随机种子
return &RandomGenerator{
rng: rand.New(rand.NewPCG(1, 2)), // 使用固定种子 (1, 2)
}
}
// 创建带自定义种子的随机数生成器
func NewRandomGeneratorWithSeed(seed uint64) *RandomGenerator {
return &RandomGenerator{
rng: rand.New(rand.NewPCG(seed, 0)),
}
}
// 创建带自定义种子的随机数生成器
func NewRandomGeneratorWithSeed2(seed1 uint64, seed2 uint64) *RandomGenerator {
return &RandomGenerator{
rng: rand.New(rand.NewPCG(seed1, seed2)),
}
}
func NewRandomGeneratorWithUnixNano() *RandomGenerator {
// 使用当前时间和纳秒作为种子
seed := time.Now().UnixNano()
return &RandomGenerator{
rng: rand.New(rand.NewPCG(uint64(seed), uint64(seed>>32))),
}
}
// RandomGenerator 的方法
func (r *RandomGenerator) IntRange(min, max int) int {
r.mu.Lock()
defer r.mu.Unlock()
if min > max {
min, max = max, min
}
return r.rng.IntN(max-min+1) + min
}
// 生成 [0, n) 范围内的公平随机整数
func (g *RandomGenerator) Int(n int) int {
g.mu.Lock()
defer g.mu.Unlock()
return g.rng.IntN(n)
}
func (r *RandomGenerator) Float() float64 {
r.mu.Lock()
defer r.mu.Unlock()
return r.rng.Float64()
}
func (r *RandomGenerator) FloatRange(min, max float64) float64 {
r.mu.Lock()
defer r.mu.Unlock()
return min + r.rng.Float64()*(max-min)
}
func (r *RandomGenerator) String(length int) string {
r.mu.Lock()
defer r.mu.Unlock()
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
b := make([]byte, length)
for i := range b {
b[i] = charset[r.rng.IntN(len(charset))]
}
return string(b)
}
func (r *RandomGenerator) Bool() bool {
r.mu.Lock()
defer r.mu.Unlock()
return r.rng.IntN(2) == 1
}
func (r *RandomGenerator) Choice(slice []string) string {
r.mu.Lock()
defer r.mu.Unlock()
if len(slice) == 0 {
return ""
}
return slice[r.rng.IntN(len(slice))]
}
func (r *RandomGenerator) Shuffle(slice []int) {
r.mu.Lock()
defer r.mu.Unlock()
r.rng.Shuffle(len(slice), func(i, j int) {
slice[i], slice[j] = slice[j], slice[i]
})
}
// 使用示例
func TestGoRandom() {
// 示例 1: 基本使用
fmt.Println("=== 示例 1: 基本使用 ===")
rg := NewRandomGenerator()
// 生成随机数
fmt.Printf("随机整数 (1-100): %d\n", rg.IntRange(1, 100))
fmt.Printf("随机浮点数: %.4f\n", rg.Float())
fmt.Printf("随机字符串: %s\n", rg.String(10))
fmt.Printf("随机布尔值: %v\n", rg.Bool())
// 示例 2: 使用固定种子(可重现的随机序列)
fmt.Println("\n=== 示例 2: 固定种子 ===")
rg1 := NewRandomGeneratorWithSeed(12345)
rg2 := NewRandomGeneratorWithSeed(12345)
now := time.Now()
seed1 := uint64(now.UnixNano())
seed2 := uint64(now.UnixNano() >> 32)
rg3 := NewRandomGeneratorWithSeed2(seed1, seed2)
fmt.Printf("生成器固定两个种子: %d, %d, %d\n", rg3.IntRange(1, 100), rg3.IntRange(1, 100), rg3.IntRange(1, 100))
// 两个生成器会产生相同的随机数序列
fmt.Printf("生成器1: %d, %d, %d\n", rg1.IntRange(1, 100), rg1.IntRange(1, 100), rg1.IntRange(1, 100))
fmt.Printf("生成器2: %d, %d, %d\n", rg2.IntRange(1, 100), rg2.IntRange(1, 100), rg2.IntRange(1, 100))
// 示例 3: 游戏应用
fmt.Println("\n=== 示例 3: 游戏应用 ===")
game := &GameExample{
rng: NewRandomGenerator(),
}
game.Play()
// 示例 4: 多个独立的生成器
fmt.Println("\n=== 示例 4: 多个独立生成器 ===")
playerRng := NewRandomGenerator() // 玩家的随机数
enemyRng := NewRandomGenerator() // 敌人的随机数
itemRng := NewRandomGenerator() // 物品的随机数
fmt.Printf("玩家伤害: %d\n", playerRng.IntRange(10, 20))
fmt.Printf("敌人伤害: %d\n", enemyRng.IntRange(5, 15))
fmt.Printf("掉落物品: %s\n", itemRng.Choice([]string{"剑", "盾", "药水", "金币"}))
// 示例 5: 并发使用
fmt.Println("\n=== 示例 5: 并发使用 ===")
concurrentExample()
}
// 游戏示例
type GameExample struct {
rng *RandomGenerator
}
func (g *GameExample) Play() {
// 掷骰子
dice := g.rng.IntRange(1, 6)
fmt.Printf("掷骰子结果: %d\n", dice)
// 暴击判定
critChance := 0.3
isCrit := g.rng.Float() < critChance
fmt.Printf("是否暴击: %v\n", isCrit)
// 随机事件
events := []string{"遇到宝箱", "遇到怪物", "发现秘密通道", "什么都没有"}
event := g.rng.Choice(events)
fmt.Printf("随机事件: %s\n", event)
}
// 并发示例 - 每个 goroutine 使用自己的生成器
func concurrentExample() {
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
// 每个 goroutine 有自己的生成器
rng := NewRandomGeneratorWithSeed(uint64(id))
for j := 0; j < 3; j++ {
num := rng.IntRange(1, 100)
fmt.Printf("Goroutine %d: %d\n", id, num)
}
}(i)
}
wg.Wait()
}
// 高级用法:带状态的随机数生成器
type StatefulRandom struct {
*RandomGenerator
callCount int
}
func NewStatefulRandom() *StatefulRandom {
return &StatefulRandom{
RandomGenerator: NewRandomGenerator(),
callCount: 0,
}
}
func (s *StatefulRandom) NextInt(min, max int) int {
s.callCount++
return s.IntRange(min, max)
}
func (s *StatefulRandom) GetCallCount() int {
return s.callCount
}
// 测试随机数分布
func testDistribution() {
rg := NewRandomGenerator()
counts := make(map[int]int)
// 生成 10000 个 1-10 的随机数
for i := 0; i < 10000; i++ {
num := rg.IntRange(1, 10)
counts[num]++
}
fmt.Println("\n=== 随机数分布测试 ===")
for i := 1; i <= 10; i++ {
fmt.Printf("%d: %d 次 (%.2f%%)\n", i, counts[i], float64(counts[i])/100)
}
}
go-utils/gorandom/gorandomnorepeat.go (2.3 KiB)
package gorandom
import "math/rand/v2"
type WeightedItem struct {
Value interface{} // 元素值
Weight float64 // 权重
}
// WeightedRandomSelector 加权随机选择器
type WeightedRandomSelector struct {
items []WeightedItem
totalWeight float64
}
// NewWeightedRandomSelector 创建新的加权随机选择器
func NewWeightedRandomSelector(items []WeightedItem) *WeightedRandomSelector {
selector := &WeightedRandomSelector{
items: make([]WeightedItem, len(items)),
}
// 复制元素并计算总权重
copy(selector.items, items)
for _, item := range selector.items {
selector.totalWeight += item.Weight
}
return selector
}
// SelectMultiple 选择多个不重复的元素
func (w *WeightedRandomSelector) SelectMultiple(count int) []WeightedItem {
if count <= 0 || count > len(w.items) {
return nil
}
// 创建可用元素的副本
availableItems := make([]WeightedItem, len(w.items))
copy(availableItems, w.items)
totalWeight := w.totalWeight
result := make([]WeightedItem, 0, count)
for i := 0; i < count; i++ {
if len(availableItems) == 0 {
break
}
// 生成随机数
randomWeight := rand.Float64() * totalWeight
// 找到对应的元素
currentWeight := 0.0
selectedIndex := -1
for j, item := range availableItems {
currentWeight += item.Weight
if randomWeight <= currentWeight {
selectedIndex = j
break
}
}
// 如果没有找到(理论上不应该发生),选择最后一个
if selectedIndex == -1 {
selectedIndex = len(availableItems) - 1
}
// 添加选中的元素到结果中
selectedItem := availableItems[selectedIndex]
result = append(result, selectedItem)
// 从可用元素中移除已选择的元素
totalWeight -= selectedItem.Weight
availableItems = append(availableItems[:selectedIndex], availableItems[selectedIndex+1:]...)
}
return result
}
// Select 选择单个元素
func (w *WeightedRandomSelector) Select() interface{} {
if len(w.items) == 0 {
return nil
}
randomWeight := rand.Float64() * w.totalWeight
currentWeight := 0.0
for _, item := range w.items {
currentWeight += item.Weight
if randomWeight <= currentWeight {
return item.Value
}
}
// 理论上不应该到达这里,但为了安全起见返回最后一个元素
return w.items[len(w.items)-1].Value
}
go-utils/gorandom/utils.go (4.7 KiB)
package gorandom
import (
cryptorand "crypto/rand"
"fmt"
"math/big"
"math/rand/v2"
"time"
)
// 不重复的随机数
func GenerateUniqueRandomInts(min, max, count int) []int {
if min >= max || count <= 0 {
return []int{}
}
total := max - min + 1
if count > total {
count = total
}
// 创建范围内所有数字
numbers := make([]int, total)
for i := 0; i < total; i++ {
numbers[i] = min + i
}
// Fisher-Yates 洗牌
for i := len(numbers) - 1; i > 0; i-- {
j := rand.IntN(i + 1)
numbers[i], numbers[j] = numbers[j], numbers[i]
}
return numbers[:count]
}
// 1. 生成随机整数(指定范围)
func RandomInt(min, max int) int {
if min > max {
min, max = max, min
}
return rand.IntN(max-min+1) + min
}
// 2. 生成随机浮点数(0.0-1.0)
func RandomFloat() float64 {
return rand.Float64()
}
// 3. 生成指定范围的随机浮点数
func RandomFloatRange(min, max float64) float64 {
return min + rand.Float64()*(max-min)
}
// 4. 生成随机字符串
func RandomString(length int) string {
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
b := make([]byte, length)
for i := range b {
b[i] = charset[rand.IntN(len(charset))]
}
return string(b)
}
// 5. 生成安全的随机字符串(使用 crypto/rand)
func SecureRandomString(length int) (string, error) {
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
b := make([]byte, length)
for i := range b {
n, err := cryptorand.Int(cryptorand.Reader, big.NewInt(int64(len(charset))))
if err != nil {
return "", err
}
b[i] = charset[n.Int64()]
}
return string(b), nil
}
// 6. 生成随机布尔值
func RandomBool() bool {
return rand.IntN(2) == 1
}
// 7. 从切片中随机选择一个元素
func RandomChoice[T any](slice []T) T {
if len(slice) == 0 {
var zero T
return zero
}
return slice[rand.IntN(len(slice))]
}
// 8. 打乱切片顺序(洗牌)
func Shuffle[T any](slice []T) {
rand.Shuffle(len(slice), func(i, j int) {
slice[i], slice[j] = slice[j], slice[i]
})
}
// 9. 生成随机密码
func RandomPassword(length int, includeSpecial bool) string {
charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
if includeSpecial {
charset += "!@#$%^&*()_+-=[]{}|;:,.<>?"
}
password := make([]byte, length)
for i := range password {
password[i] = charset[rand.IntN(len(charset))]
}
return string(password)
}
// 10. 生成随机十六进制字符串
func RandomHex(length int) string {
const hex = "0123456789abcdef"
b := make([]byte, length)
for i := range b {
b[i] = hex[rand.IntN(16)]
}
return string(b)
}
// 12. 按权重随机选择
func WeightedRandomChoice[T any](items []T, weights []int) T {
if len(items) == 0 || len(items) != len(weights) {
var zero T
return zero
}
totalWeight := 0
for _, w := range weights {
totalWeight += w
}
r := rand.IntN(totalWeight)
for i, w := range weights {
r -= w
if r < 0 {
return items[i]
}
}
return items[len(items)-1]
}
// 13. 生成正态分布随机数
func RandomNormal(mean, stdDev float64) float64 {
return rand.NormFloat64()*stdDev + mean
}
// 14. 生成随机颜色(RGB)
func RandomColor() string {
return fmt.Sprintf("#%02x%02x%02x",
rand.IntN(256),
rand.IntN(256),
rand.IntN(256))
}
// 15. 生成随机日期时间
func RandomDateTime(start, end time.Time) time.Time {
delta := end.Unix() - start.Unix()
sec := rand.Int64N(delta) + start.Unix()
return time.Unix(sec, 0)
}
// 示例用法
func Test() {
// 注意:Go 1.20+ 不再需要手动设置种子
// 旧版本需要:rand.Seed(time.Now().UnixNano())
// 1. 随机整数
fmt.Printf("Random int (1-100): %d\n", RandomInt(0, 1))
// 2. 随机浮点数
fmt.Printf("Random float: %.4f\n", RandomFloat())
// 3. 随机字符串
fmt.Printf("Random string: %s\n", RandomString(10))
// 4. 安全随机字符串
secureStr, _ := SecureRandomString(16)
fmt.Printf("Secure random string: %s\n", secureStr)
// 5. 随机选择
fruits := []string{"apple", "banana", "orange", "grape"}
fmt.Printf("Random fruit: %s\n", RandomChoice(fruits))
// 6. 洗牌
numbers := []int{1, 2, 3, 4, 5}
Shuffle(numbers)
fmt.Printf("Shuffled numbers: %v\n", numbers)
// 7. 随机密码
fmt.Printf("Random password: %s\n", RandomPassword(12, true))
// 9. 加权随机
items := []string{"common", "rare", "epic", "legendary"}
weights := []int{70, 20, 8, 2}
fmt.Printf("Weighted random: %s\n", WeightedRandomChoice(items, weights))
// 10. 随机颜色
fmt.Printf("Random color: %s\n", RandomColor())
// 11. 随机日期
start := time.Now().AddDate(-1, 0, 0)
end := time.Now()
fmt.Printf("Random date: %s\n", RandomDateTime(start, end).Format("2006-01-02"))
}
go-utils/gotime/readme.md (33 B)
时间日期相关工具函数
go-utils/gotime/time.go (3.1 KiB)
package gotime
import "time"
// 当前时区相关日期函数
func DateTime(format string) string {
return time.Now().Format(format)
}
func Today(format ...string) string {
if len(format) > 0 {
return DateTime(format[0])
}
return DateTime(time.DateOnly)
}
func Now(format ...string) string {
if len(format) > 0 {
return DateTime(format[0])
}
return DateTime(time.DateTime)
}
func NextDate(d int, format ...string) string {
if len(format) > 0 {
return time.Now().AddDate(0, 0, d).Format(format[0])
}
return time.Now().AddDate(0, 0, d).Format(time.DateOnly)
}
func Yesterday(format ...string) string {
return NextDate(-1, format...)
}
func Ts2Date(ts int64, format ...string) string {
if len(format) > 0 {
return time.Unix(ts, 0).Format(format[0])
}
return time.Unix(ts, 0).Format(time.DateOnly)
}
func Ts2DateTime(ts int64, format ...string) string {
if len(format) > 0 {
return time.Unix(ts, 0).Format(format[0])
}
return time.Unix(ts, 0).Format(time.DateTime)
}
func Date2Ts(date string) int64 {
ti, _ := time.ParseInLocation(time.DateOnly, date, time.Local)
return ti.Unix()
}
func DateTime2Ts(dateTime string) int64 {
return DateTime2TsLocalFormat(dateTime, time.Local, time.DateTime)
}
//--------------本地化时间--------------
//没有format 时日志格式为 time.DateOnly 和 time.DateTime
// 当前时间通用 格式用法
func DateTimeLocal(format string, timeOffsetSec int) string {
return Ts2DateTimeLocalFormat(time.Now().Unix(), timeOffsetSec, format)
}
// NowDate
func TodayLocal(timeOffsetSec int) string {
return Ts2DateLocal(time.Now().Unix(), timeOffsetSec)
}
// NowDateTime
func NowLocal(timeOffsetSec int) string {
return Ts2DateTimeLocal(time.Now().Unix(), timeOffsetSec)
}
// NextDate
func NextDateLocal(d int, timeOffsetSec int) string {
location := time.FixedZone("OffsetZone", timeOffsetSec)
t := time.Unix(time.Now().Unix(), 0)
t.In(location)
return Ts2DateTimeLocalFormat(t.AddDate(0, 0, d).Unix(), timeOffsetSec, time.DateOnly)
}
// 本地化日期 给定时间戳(以秒为单位)
func Ts2DateLocal(ts int64, timeOffsetSec int) string {
return Ts2DateTimeLocalFormat(ts, timeOffsetSec, time.DateOnly)
}
// 本地化日期时间 给定时间戳(以秒为单位)
func Ts2DateTimeLocal(ts int64, timeOffsetSec int) string {
return Ts2DateTimeLocalFormat(ts, timeOffsetSec, time.DateTime)
}
func Date2TsLocal(date string, location *time.Location) int64 {
return DateTime2TsLocalFormat(date, location, time.DateOnly)
}
func DateTime2TsLocal(dateTime string, location *time.Location) int64 {
return DateTime2TsLocalFormat(dateTime, location, time.DateTime)
}
func DateTime2TsLocalFormat(dateTime string, location *time.Location, format string) int64 {
ti, _ := time.ParseInLocation(format, dateTime, location)
return ti.Unix()
}
// 将时间对象转换为特定时区的时间
func Ts2DateTimeLocalFormat(ts int64, timeOffsetSec int, format string) string {
t := time.Unix(ts, 0)
location := time.FixedZone("OffsetZone", timeOffsetSec)
// 将时间对象转换为特定时区的时间
localTime := t.In(location)
return localTime.Format(format)
}
go-utils/gotime/timer.go (2.3 KiB)
package gotime
import (
goutils "github.com/gif-gif/go.io/go-utils"
"sync"
"time"
)
type Timer struct {
duration time.Duration
stop chan struct{}
pause chan struct{}
resume chan struct{}
reset chan time.Duration
force chan struct{}
wg sync.WaitGroup
isPaused bool
isStopped bool
mu sync.Mutex
ticker *time.Ticker
}
func NewTimer(duration time.Duration) *Timer {
return &Timer{
duration: duration,
// 使用buffered channels避免阻塞
stop: make(chan struct{}, 1),
pause: make(chan struct{}, 1),
resume: make(chan struct{}, 1),
reset: make(chan time.Duration, 1),
force: make(chan struct{}, 1),
isStopped: true,
}
}
// 不要重复调用Start方法
func (t *Timer) Start(task func()) {
t.mu.Lock()
// 重置停止状态
t.isStopped = false
t.mu.Unlock()
t.wg.Add(1)
if t.ticker == nil {
t.ticker = time.NewTicker(t.duration)
}
goutils.AsyncFunc(func() {
defer t.wg.Done()
for {
if t.ticker == nil {
return
}
select {
case <-t.ticker.C:
t.mu.Lock()
isPaused := t.isPaused
t.mu.Unlock()
if !isPaused {
task()
}
case <-t.pause:
t.mu.Lock()
t.isPaused = true
t.mu.Unlock()
case <-t.resume:
t.mu.Lock()
t.isPaused = false
t.mu.Unlock()
case newDuration := <-t.reset:
t.ticker.Reset(newDuration)
t.duration = newDuration
case <-t.force:
task()
case <-t.stop:
return
}
}
})
}
func (t *Timer) Stop() {
t.mu.Lock()
if !t.isStopped {
t.isStopped = true
if t.ticker != nil {
t.ticker.Stop()
}
close(t.stop)
// 重新创建stop channel为下次使用准备
t.stop = make(chan struct{}, 1)
t.ticker = nil
}
t.mu.Unlock()
t.wg.Wait()
}
func (t *Timer) Pause() {
t.mu.Lock()
if !t.isStopped && !t.isPaused {
select {
case t.pause <- struct{}{}:
default:
}
}
t.mu.Unlock()
}
func (t *Timer) Resume() {
t.mu.Lock()
if !t.isStopped && t.isPaused {
select {
case t.resume <- struct{}{}:
default:
}
}
t.mu.Unlock()
}
func (t *Timer) ForceRun() {
t.mu.Lock()
if !t.isStopped {
select {
case t.force <- struct{}{}:
default:
}
}
t.mu.Unlock()
}
func (t *Timer) Reset(duration time.Duration) {
t.mu.Lock()
if !t.isStopped {
select {
case t.reset <- duration:
default:
}
}
t.mu.Unlock()
}
go-utils/gotime/timex.go (8.7 KiB)
package gotime
import (
"errors"
"fmt"
"math"
"regexp"
"strings"
"time"
)
var (
TimeLayout string = "2006-01-02 15:04:05"
DateLayout = "20060102"
TimeFormat = map[string]string{
"Y-m-d H:i:s": "2006-01-02 15:04:05",
"Y-m-d": "2006-01-02",
"Ymd": "20060102",
"H:i:s": "15:04:05",
"Y": "2006",
"m": "01",
"d": "02",
}
)
// GetTimeNow 获取当前时间GetTimeNow(),用于测试时的时间修改
func GetTimeNow() time.Time {
//redisConf := redis.RedisKeyConf{
// RedisConf: redis.RedisConf{
// Host: "122.228.113.235:17006",
// Type: "",
// Pass: "xiaozi527sport",
// TLS: false,
// },
//}
//redisClient := redisConf.NewRedis()
//val, err := redisClient.Get("test:now:gap:seconds")
//if err != nil {
// fmt.Errorf("GetTimeNow test:now:gap error:%s", err.Error())
// return time.Now()
//}
//gap, _ := strconv.ParseInt(val, 10, 64)
//return time.Now().Add(time.Second * time.Duration(gap))
return time.Now()
}
func Ts2Time(t int64) time.Time {
return time.Unix(t, 0)
}
func BeijingTimeLocation() *time.Location {
loc, _ := time.LoadLocation("Asia/Shanghai")
return loc
}
func GetChinaTomorrowAMSeconds(isBeijing bool) int64 {
now := GetTimeNow()
if isBeijing {
loc, _ := time.LoadLocation("Asia/Shanghai")
t, _ := time.ParseInLocation("2006-01-02", now.AddDate(0, 0, 1).Format("2006-01-02"), loc)
secondsF := t.Sub(GetTimeNow()).Seconds()
return int64(secondsF)
} else {
secondsF := now.Sub(GetTimeNow()).Seconds()
return int64(secondsF)
}
}
func GetLocalTomorrowAMSeconds() int64 {
now := GetTimeNow()
t, _ := time.ParseInLocation("2006-01-02", now.AddDate(0, 0, 1).Format("2006-01-02"), time.Local)
secondsF := t.Sub(GetTimeNow()).Seconds()
return int64(secondsF)
}
func GetTodayZero() time.Time {
t := GetTimeNow()
zero := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
return zero
}
func GetZero(targetTime time.Time) time.Time {
zero := time.Date(targetTime.Year(), targetTime.Month(), targetTime.Day(), 0, 0, 0, 0, targetTime.Location())
return zero
}
// ParseTime 解析时间,"2021-03-17 00:00:00"
func ParseTime(timeStr string) (datetime time.Time) {
datetime, _ = time.ParseInLocation(TimeLayout, timeStr, time.Local)
return
}
//eg:20210812170000
func ParseTimeString(timeStr string) (datetime time.Time) {
datetime, _ = time.ParseInLocation("20060102150405", timeStr, time.Local)
return
}
func GetDateInterval(t1, t2 time.Time) int {
t1 = time.Date(t1.Year(), t1.Month(), t1.Day(), 0, 0, 0, 0, time.Local)
t2 = time.Date(t2.Year(), t2.Month(), t2.Day(), 0, 0, 0, 0, time.Local)
interval := int(math.Abs(t1.Sub(t2).Hours())/24) + 1
return interval
}
// SinceDays 获取过去的天数,dateString格式20060102
func SinceDays(dateString string) (int64, error) {
return SinceDaysEx(dateString, DateLayout)
}
// SinceDays 获取过去的天数
func SinceDaysEx(dateString string, format string) (int64, error) {
targetTime, err := time.ParseInLocation(format, dateString, time.Local)
if err != nil {
return 0, err
} else {
days := int64(math.Ceil(time.Since(targetTime).Hours() / 24))
if days == 0 {
days = 1
}
return days, nil
}
}
func IsSameDay(t1, t2 time.Time) bool {
year1, month1, day1 := t1.Date()
year2, month2, day2 := t2.Date()
return day1 == day2 && month1 == month2 && year1 == year2
}
// LastHourStartAndEnd 上一个小时的开始和结束时间戳
func LastHourStartAndEnd(isBeijing bool) (int, int64, int64) {
now1 := time.Now()
var now time.Time
if isBeijing {
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
fmt.Println("Error loading location:", err)
return 0, 0, 0
}
now = now1.In(location)
} else {
now = now1
}
startOfLastHour := time.Date(now.Year(), now.Month(), now.Day(), now.Hour()-1, 0, 0, 0, now.Location())
endOfLastHour := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location()).Add(-time.Nanosecond)
startOfLastHourUnix := startOfLastHour.Unix()
endOfLastHourUnix := endOfLastHour.Unix()
return startOfLastHour.Hour(), startOfLastHourUnix, endOfLastHourUnix
}
func CurrentHourStartAndEnd() (int, int64, int64) {
now := time.Now()
startOfHour := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
endOfHour := startOfHour.Add(time.Hour - time.Second)
startOfHourTimestamp := startOfHour.Unix()
endOfHourTimestamp := endOfHour.Unix()
return startOfHour.Hour(), startOfHourTimestamp, endOfHourTimestamp
}
func GetNowDateForLocation(isBeijing bool) string {
tm := time.Now()
var tmInLocation time.Time
if isBeijing {
// 加载时区
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
fmt.Println("Error loading location:", err)
return ""
}
// 将时间转换为指定时区
tmInLocation = tm.In(location)
} else {
tmInLocation = tm
}
// 将时间格式化为日期字符串
dateStr := tmInLocation.Format(time.DateOnly)
return dateStr
}
func ToDate(ts int64, isBeijing bool) string {
tm := time.Unix(ts, 0)
var tmInLocation time.Time
if isBeijing {
// 加载时区
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
fmt.Println("Error loading location:", err)
return ""
}
// 将时间转换为指定时区
tmInLocation = tm.In(location)
} else {
tmInLocation = tm
}
// 将时间格式化为日期字符串
dateStr := tmInLocation.Format(time.DateOnly)
return dateStr
}
func ToDateTime(ts int64, isBeijing bool) string {
tm := time.Unix(ts, 0)
var tmInLocation time.Time
if isBeijing {
// 加载时区
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
fmt.Println("Error loading location:", err)
return ""
}
// 将时间转换为指定时区
tmInLocation = tm.In(location)
} else {
tmInLocation = tm
}
// 将时间格式化为日期字符串
dateStr := tmInLocation.Format(time.DateTime)
return dateStr
}
// 返回相差天数
func TimeRangeDay(stTime int64, endTIme int64) int {
startTime := time.Unix(stTime, 0)
endTime := time.Unix(endTIme, 0)
durationDays := int(endTime.Sub(startTime).Hours()/24 + 1)
return durationDays
}
func TimeRangeDates(startDate string, endDate string) []string {
//startDate := "2021-01-01"
//endDate := "2021-01-10"
dates := []string{}
start, _ := time.Parse("2006-01-02", startDate)
end, _ := time.Parse("2006-01-02", endDate)
for d := start; !d.After(end); d = d.AddDate(0, 0, 1) {
fmt.Println(d.Format("2006-01-02"))
dates = append(dates, d.Format("2006-01-02"))
}
return dates
}
func DatesForRangeTs(startTimestamp int64, endTimestamp int64, format string) []string {
if format == "" {
format = "2006-01-02"
}
// 转换为time.Time对象
startTime := time.Unix(startTimestamp, 0)
endTime := time.Unix(endTimestamp, 0)
// 创建一个当前时间的副本
currentTime := startTime
dates := []string{}
// 循环直到达到结束时间
for currentTime.Before(endTime) {
// 格式化输出日期
dates = append(dates, currentTime.Format(format))
// 增加一天
currentTime = currentTime.AddDate(0, 0, 1)
}
// 输出最后一天(如果最后一天的时间戳在当天内)
if currentTime.Format(format) == endTime.Format(format) {
dates = append(dates, currentTime.Format(format))
}
return dates
}
// convertTo24HourFormat 将时间字符串转换为 0-23 小时制
// timeStr: 时间字符串,例如 "3:04 PM"
// 3:04 PM-->15:04 ,必须是这两个格式之一
func ConvertAmPmHourTo24HourFormat(timeStr string, layout24Hour string) (string, error) {
// 定义时间解析格式
const layout12Hour = "3:04 PM"
if layout24Hour == "" {
layout24Hour = "15:04"
}
if !strings.HasPrefix(layout24Hour, "15") {
return "", errors.New("layout24Hour must start with 15 or 15:04")
}
// 解析时间字符串
t, err := time.Parse(layout12Hour, timeStr)
if err != nil {
return "", fmt.Errorf("failed to parse time: %v", err)
}
// 格式化为 24 小时制
return t.Format(layout24Hour), nil
}
// 格式为: 2021-08-06T07:00:00+0000 To time.Time
func ConvertToGMTTime(gmtTime string) (time.Time, error) {
t, err := time.Parse("2006-01-02T15:04:05-0700", gmtTime)
return t, err
}
// 判断是不是 2024-11-22 4 格式的时间串
func IsValidDateTime(str string) bool {
// 定义日期时间格式的正则表达式,忽略多余空格
re := regexp.MustCompile(`^\d{4}-\d{1,2}-\d{1,2}\s+\d{1,2}$`)
// 使用正则表达式匹配字符串
isValidFormat := re.MatchString(strings.TrimSpace(str))
return isValidFormat
}
func IsValidDate(str string) bool {
// 定义日期时间格式的正则表达式,忽略多余空格
re := regexp.MustCompile(`^\d{4}-\d{1,2}-\d{1,2}$`)
// 使用正则表达式匹配字符串
isValidFormat := re.MatchString(strings.TrimSpace(str))
return isValidFormat
}
go-utils/gozip/gozip.go (4.1 KiB)
package gozip
import (
"bytes"
"compress/gzip"
"github.com/andybalholm/brotli"
"io"
"os"
)
// xNlContentEncoding: br, gzip
const (
XNlContentEncoding = "X-NL-Content-Encoding" //默认Header 压缩标识
NOZIP = "nozip" //不压缩
GZIP = "gzip"
BR = "br"
GoZipNoType = "__nozip__" //默认Header 压缩标识
GoZipType = "__zip__" //默认Header 压缩标识
UnGoZipType = "__unzip__" //默认Header 压缩标识
)
func GZip(data []byte) ([]byte, error) {
// 创建一个buffer用于存储压缩后的数据
var buf bytes.Buffer
// 创建一个gzip writer
gzipWriter := gzip.NewWriter(&buf)
// 写入数据
_, err := gzipWriter.Write(data)
if err != nil {
return nil, err
}
// 关闭writer,确保所有数据被刷新到buffer中
if err := gzipWriter.Close(); err != nil {
return nil, err
}
// 返回压缩后的数据
return buf.Bytes(), nil
}
func UnGZip(compressedData []byte) ([]byte, error) {
// 创建一个byte reader
bytesReader := bytes.NewReader(compressedData)
// 创建一个gzip reader
gzipReader, err := gzip.NewReader(bytesReader)
if err != nil {
return nil, err
}
defer gzipReader.Close()
// 读取解压后的数据
var buf bytes.Buffer
if _, err := io.Copy(&buf, gzipReader); err != nil {
return nil, err
}
// 返回解压后的数据
return buf.Bytes(), nil
}
func GZipFile(src, dst string) error {
// 打开源文件
sourceFile, err := os.Open(src)
if err != nil {
return err
}
defer sourceFile.Close()
// 创建目标文件
destFile, err := os.Create(dst)
if err != nil {
return err
}
defer destFile.Close()
// 创建gzip writer
gzipWriter := gzip.NewWriter(destFile)
defer gzipWriter.Close()
// 从源文件复制到gzip writer
_, err = io.Copy(gzipWriter, sourceFile)
return err
}
func UnGZipFile(src, dst string) error {
// 打开源文件(gzip压缩文件)
sourceFile, err := os.Open(src)
if err != nil {
return err
}
defer sourceFile.Close()
// 创建gzip reader
gzipReader, err := gzip.NewReader(sourceFile)
if err != nil {
return err
}
defer gzipReader.Close()
// 创建目标文件
destFile, err := os.Create(dst)
if err != nil {
return err
}
defer destFile.Close()
// 从gzip reader复制到目标文件
_, err = io.Copy(destFile, gzipReader)
return err
}
// BR 压缩
// 压缩数据
func BrZip(data []byte, quality int) ([]byte, error) {
var buf bytes.Buffer
// 创建brotli writer,quality参数范围为0-11,值越大压缩率越高但更慢
brWriter := brotli.NewWriterLevel(&buf, quality)
// 写入数据
_, err := brWriter.Write(data)
if err != nil {
return nil, err
}
// 关闭writer,确保所有数据被写入
if err := brWriter.Close(); err != nil {
return nil, err
}
// 返回压缩后的数据
return buf.Bytes(), nil
}
// 解压数据
func UnBrZip(compressedData []byte) ([]byte, error) {
// 创建brotli reader
brReader := brotli.NewReader(bytes.NewReader(compressedData))
// 读取解压后的数据
var buf bytes.Buffer
if _, err := io.Copy(&buf, brReader); err != nil {
return nil, err
}
// 返回解压后的数据
return buf.Bytes(), nil
}
func BrZipFile(src, dst string, quality int) error {
// 打开源文件
sourceFile, err := os.Open(src)
if err != nil {
return err
}
defer sourceFile.Close()
// 创建目标文件
destFile, err := os.Create(dst)
if err != nil {
return err
}
defer destFile.Close()
// 创建brotli writer
brWriter := brotli.NewWriterLevel(destFile, quality)
defer brWriter.Close()
// 从源文件复制到brotli writer
_, err = io.Copy(brWriter, sourceFile)
return err
}
func UnBrZipFile(src, dst string) error {
// 打开源文件(brotli压缩文件)
sourceFile, err := os.Open(src)
if err != nil {
return err
}
defer sourceFile.Close()
// 创建brotli reader
brReader := brotli.NewReader(sourceFile)
// 创建目标文件
destFile, err := os.Create(dst)
if err != nil {
return err
}
defer destFile.Close()
// 从brotli reader复制到目标文件
_, err = io.Copy(destFile, brReader)
return err
}
go-utils/gozip/types.go (14 B)
package gozip
go-utils/gozip/utils.go (1.2 KiB)
package gozip
import (
"github.com/andybalholm/brotli"
goerror "github.com/gif-gif/go.io/go-error"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
)
func Compress(body []byte, compressMethod string, compressType string) (bool, []byte, error) {
defer goutils.Recovery(func(err any) {
golog.Warn(err)
})
if compressType == "" {
compressType = UnGoZipType
}
if compressMethod == "" { //没有压缩逻辑
return false, body, nil
}
var data []byte
var err error
if compressMethod == GZIP { //压缩和解压
if compressType == GoZipType {
data, err = GZip(body)
if err != nil {
return false, nil, goerror.NewParamErrMsg("GZip error ")
}
} else {
data, err = UnGZip(body)
if err != nil {
return false, nil, goerror.NewParamErrMsg("UnGZip error ")
}
}
} else if compressMethod == BR {
if compressType == GoZipType { //压缩和解压
data, err = BrZip(body, brotli.BestCompression)
if err != nil {
return false, nil, goerror.NewParamErrMsg("BrZip error ")
}
} else {
data, err = UnBrZip(body)
if err != nil {
return false, nil, goerror.NewParamErrMsg("UnBrZip error ")
}
}
} else { //不 压缩和解压
data = body
}
return true, data, nil
}
go-utils/id_gen.go (480 B)
package goutils
import (
"github.com/google/uuid"
"strconv"
"sync"
)
type iGenId interface {
GenId() int64
}
var (
__genId iGenId
__genIdOne sync.Once
)
func GenIdInit(adapter iGenId) {
__genId = adapter
}
func GenId() int64 {
__genIdOne.Do(func() {
if __genId == nil {
__genId = &SnowFlakeId{WorkerId: 1}
}
})
return __genId.GenId()
}
func GenIdStr() string {
return strconv.FormatInt(GenId(), 10)
}
func UUID() string {
return uuid.New().String()
}
go-utils/id_snow_flake.go (2.0 KiB)
package goutils
import (
"sync"
"time"
)
//缺点:但是雪花算法强依赖机器时钟,如果机器上时钟回拨,会导致发号重复或者服务会处于不可用状态。如果恰巧回退前生成过一些ID,而时间回退后,生成的ID就有可能重复。官方对于此并没有给出解决方案,而是简单的抛错处理,这样会造成在时间被追回之前的这段时间服务不可用。很多其他类雪花算法也是在此思想上的设计然后改进规避它的缺陷,
//后面介绍的百度 UidGenerator 和 美团分布式ID生成系统 Leaf 中snowflake模式都是在 snowflake 的基础上演进出来的。#
// Snowflake,雪花算法是由Twitter开源的分布式ID生成算法,以划分命名空间的方式将 64-bit位分割成多个部分,
// 每个部分代表不同的含义。这种就是将64位划分为不同的段,每段代表不同的涵义,基本就是时间戳、机器ID和序列数。
// 为什么如此重要?因为它提供了一种ID生成及生成的思路,当然这种方案就是需要考虑时钟回拨的问题以及做一些 buffer的缓冲设计提高性能。
// 雪花算法
type SnowFlakeId struct {
DataCenterId int // 机房或者数据中心ID 0 - 31
WorkerId int // 机器或者容器ID 0 - 31
lastTime int64 // 最后生成ID时间
sn int // 毫秒内序列序号 0-4095
mu sync.Mutex
}
func (sf *SnowFlakeId) GenId() int64 {
sf.mu.Lock()
defer sf.mu.Unlock()
ts := time.Now().UnixNano() / 1e6
if sf.lastTime == ts {
// 2的12次方 -1 = 4095,每毫秒可产生4095个ID
if sf.sn > 4095 {
time.Sleep(time.Millisecond)
ts = time.Now().UnixNano() / 1e6
sf.sn = 0
}
} else {
sf.sn = 0
}
sf.sn += 1
sf.lastTime = ts
// 时间戳,向左移动22位
ts = ts << 22
// 机房ID,向左移动17位
dataCenterId := sf.DataCenterId << 17
// 机器ID,向左移动12位
machineId := sf.WorkerId << 12
return ts | int64(dataCenterId) | int64(machineId) | int64(sf.sn)
}
go-utils/id_snow_flake_test.go (967 B)
package goutils
import (
"fmt"
goredis "github.com/gif-gif/go.io/go-db/go-redis"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gogf/gf/util/gconv"
"testing"
)
// Name string `yaml:"Name" json:"name,optional"`
// Addr string `yaml:"Addr" json:"addr,optional"`
// Password string `yaml:"Password" json:"password,optional"`
// DB int `yaml:"DB" json:"db,optional"`
// Prefix string `yaml:"Prefix" json:"prefix,optional"`
func TestGenId(t *testing.T) {
goredis.Init(goredis.Config{
Name: "test",
Addr: "127.0.0.1:6379",
Password: "",
DB: 0,
Prefix: "snowFake",
})
GenIdInit(&SnowFlakeId{WorkerId: 2})
for i := 0; i < 50; i++ {
gid := GenIdStr()
b := goredis.Default().HExists("test", gconv.String(gid)).Val()
if b {
golog.WithTag("snowFlake").Info("my god")
} else {
goredis.Default().HSet("test", gconv.String(gid), "1")
}
fmt.Println(gid)
}
golog.WithTag("snowFlake").Info("The End")
}
go-utils/idcode.go (2.2 KiB)
package goutils
import (
"bytes"
"errors"
"math/rand"
"strconv"
"strings"
"time"
)
const key = "6A7CDKV5TBH0ULFSEP82XMW1G9R3YJQNZ"
type idCode struct {
base int64
key string
l int
}
func NewIdCode(key string) *idCode {
return &idCode{
base: 100000,
key: key,
l: len(key),
}
}
/**
* <随机字符A><加密字符串><验证字符B>
* 随机字符A = 从key里面随机获取一个字符
* 加密字符串 = 每位转换成16进制的ID数字+随机数,然后取余key的长度,得到的数字就是该位在key的字符,加入到总加密后的字符串中
* 验证字符B = 从key里面获取一个字符,字符位置=(密钥长度-随机数+给定ID长度)%密钥长度
*/
func (c *idCode) Encode(id int64) string {
id += c.base
n := c.randNum()
hexArr := []rune(strconv.FormatInt(id, 16))
keyArr := []rune(c.key)
var buf bytes.Buffer
buf.WriteRune(keyArr[n])
for _, h := range hexArr {
hi, _ := strconv.ParseInt(string(h), 16, 64)
offset := int(hi) + n
buf.WriteRune(keyArr[offset%c.l])
}
buf.WriteRune(keyArr[(c.l-n+len(strconv.FormatInt(id, 10)))%c.l])
return buf.String()
}
/**
* <随机字符A><加密字符串><验证字符B>
* 1. 根据随机字符A,获取随机数n
* 2. 获取中间字符串,计算得到id
* 3. 验证验证字符B是否正确
*/
func (c *idCode) Decode(str string) (id int64, err error) {
if str == "" {
err = errors.New("code为空")
return
}
strArr := []rune(str)
keyArr := []rune(c.key)
l := len(strArr)
n := strings.IndexRune(c.key, strArr[0])
var buf bytes.Buffer
for _, s := range strArr[1 : l-1] {
pos := strings.IndexRune(c.key, s)
if pos >= n {
buf.WriteString(strconv.FormatInt(int64(pos-n), 16))
} else {
buf.WriteString(strconv.FormatInt(int64(c.l-n+pos), 16))
}
}
id, err = strconv.ParseInt(buf.String(), 16, 64)
if err != nil {
return
}
if strArr[l-1] != keyArr[(c.l-n+len(strconv.FormatInt(id, 10)))%c.l] {
err = errors.New("校验错误")
return
}
id -= c.base
return
}
func (c *idCode) randNum() int {
rand.Seed(time.Now().UnixNano())
return rand.Intn(c.l - 1)
}
func Id2Code(id int64) string {
return NewIdCode(key).Encode(id)
}
func Code2Id(code string) (int64, error) {
return NewIdCode(key).Decode(code)
}
go-utils/idcode_test.go (583 B)
package goutils
import (
"fmt"
"math/rand"
"testing"
"time"
)
func TestId2Code(t *testing.T) {
for i := 1; i < 1000; i++ {
code := Id2Code(int64(i))
id, _ := Code2Id(code)
fmt.Println(id, code)
}
}
func TestKey(t *testing.T) {
key := []rune("123567890ABCDEFGHJKLMNPQRSTUVWXYZ")
rand.Seed(time.Now().UnixNano())
var data []rune
for {
l := len(key)
if l == 0 {
break
}
if l == 1 {
data = append(data, key[0])
break
}
n := rand.Intn(l - 1)
data = append(data, key[n])
key = append(key[:n], key[n+1:]...)
}
fmt.Println(string(data))
}
go-utils/map.go (269 B)
package goutils
import "encoding/json"
type M map[string]interface{}
func (m M) Json() []byte {
b, _ := json.Marshal(&m)
return b
}
func (m M) String() string {
return string(m.Json())
}
func (m M) Params() Params {
p, _ := Byte(m.Json()).Params()
return p
}
go-utils/noncestr.go (290 B)
package goutils
import (
"crypto/rand"
"encoding/hex"
"io"
)
func NonceStr() string {
bf := make([]byte, 8)
io.ReadFull(rand.Reader, bf)
return hex.EncodeToString(bf)
}
func NonceStr8() string {
bf := make([]byte, 4)
io.ReadFull(rand.Reader, bf)
return hex.EncodeToString(bf)
}
go-utils/params.go (3.2 KiB)
package goutils
import (
"encoding/json"
golog "github.com/gif-gif/go.io/go-log"
"reflect"
"strconv"
"strings"
)
type Byte []byte
func (b Byte) Params() (p Params, err error) {
return Json2Params(b)
}
type Params struct {
data interface{}
}
func NewParams() Params {
return Params{data: map[string]interface{}{}}
}
func Json2Params(b []byte) (p Params, err error) {
p = NewParams()
if err = json.Unmarshal(b, &p.data); err != nil {
golog.WithField("params", string(b)).Error(err)
}
return
}
func (p Params) Set(key string, val interface{}) Params {
p.data.(map[string]interface{})[key] = val
return p
}
func (p Params) Get(key string) Params {
keys := strings.Split(key, ".")
for _, k := range keys {
if data, ok := (p.data).(map[string]interface{}); ok {
if v, ok := data[k]; ok {
p.data = v
continue
}
p.data = nil
break
}
p.data = nil
break
}
return p
}
func (p Params) String() string {
switch reflect.ValueOf(p.data).Kind() {
case reflect.Float64:
return strconv.FormatFloat(p.Float64(), 'f', -1, 64)
case reflect.Float32:
return strconv.FormatFloat(p.Float64(), 'f', -1, 64)
case reflect.Int:
return strconv.FormatInt(p.Int64(), 10)
case reflect.Int32:
return strconv.FormatInt(p.Int64(), 10)
case reflect.Int64:
return strconv.FormatInt(p.Int64(), 10)
case reflect.Bool:
return strconv.FormatBool(p.Bool())
case reflect.String:
v := (p.data).(string)
if v == "null" {
v = ""
}
return v
}
v := string(p.JSON())
if v == "null" {
v = ""
}
return v
}
func (p Params) Int64() int64 {
switch reflect.ValueOf(p.data).Kind() {
case reflect.Float64:
return int64((p.data).(float64))
case reflect.Float32:
return int64((p.data).(float32))
case reflect.Int:
return int64((p.data).(int))
case reflect.Int32:
return int64((p.data).(int32))
case reflect.Int64:
return (p.data).(int64)
case reflect.Bool:
if (p.data).(bool) {
return 1
}
case reflect.String:
v, _ := strconv.ParseInt((p.data).(string), 10, 64)
return v
}
return 0
}
func (p Params) Int32() int32 {
return int32(p.Int64())
}
func (p Params) Int() int {
return int(p.Int64())
}
func (p Params) Float64() float64 {
if v, ok := (p.data).(float64); ok {
return v
}
return 0
}
func (p Params) Float32() float32 {
if v, ok := (p.data).(float32); ok {
return v
}
return 0
}
func (p Params) Bool() bool {
if v, ok := (p.data).(bool); ok {
return v
}
return false
}
func (p Params) Array() (ps []Params) {
ps = []Params{}
if arr, ok := (p.data).([]interface{}); ok {
for _, data := range arr {
ps = append(ps, Params{data: data})
}
}
return
}
func (p Params) Map() (rst map[string]Params) {
rst = map[string]Params{}
if m, ok := (p.data).(map[string]interface{}); ok {
for k, data := range m {
rst[k] = Params{data: data}
}
}
return
}
func (p Params) Data() interface{} {
return p.data
}
func (p Params) MapData() map[string]interface{} {
if data, ok := (p.data).(map[string]interface{}); ok {
return data
}
return map[string]interface{}{}
}
func (p Params) ArrayData() []interface{} {
if data, ok := (p.data).([]interface{}); ok {
return data
}
return []interface{}{}
}
func (p Params) JSON() []byte {
buf, _ := json.Marshal(p.data)
return buf
}
go-utils/pinyin.go (3.4 KiB)
package goutils
import (
"errors"
"strconv"
"strings"
"sync"
"unicode/utf8"
)
var (
ErrInitialize = errors.New("not yet initialized")
)
var (
tones = [][]rune{
{'ā', 'ē', 'ī', 'ō', 'ū', 'ǖ', 'Ā', 'Ē', 'Ī', 'Ō', 'Ū', 'Ǖ'},
{'á', 'é', 'í', 'ó', 'ú', 'ǘ', 'Á', 'É', 'Í', 'Ó', 'Ú', 'Ǘ'},
{'ǎ', 'ě', 'ǐ', 'ǒ', 'ǔ', 'ǚ', 'Ǎ', 'Ě', 'Ǐ', 'Ǒ', 'Ǔ', 'Ǚ'},
{'à', 'è', 'ì', 'ò', 'ù', 'ǜ', 'À', 'È', 'Ì', 'Ò', 'Ù', 'Ǜ'},
}
neutrals = []rune{'a', 'e', 'i', 'o', 'u', 'v', 'A', 'E', 'I', 'O', 'U', 'V'}
)
var (
// 从带声调的声母到对应的英文字符的映射
tonesMap map[rune]rune
// 从汉字到声调的映射
numericTonesMap map[rune]int
// 从汉字到拼音的映射(带声调)
pinyinMap map[rune]string
initialized bool
)
type Mode int
const (
WithoutTone Mode = iota + 1 // 默认模式,例如:guo
Tone // 带声调的拼音 例如:guó
InitialsInCapitals // 首字母大写不带声调,例如:Guo
)
type pinyin struct {
origin string
split string
mode Mode
}
func PinYin(origin string) (string, error) {
new(sync.Once).Do(func() {
tonesMap = make(map[rune]rune)
numericTonesMap = make(map[rune]int)
pinyinMap = make(map[rune]string)
for i, runes := range tones {
for j, tone := range runes {
tonesMap[tone] = neutrals[j]
numericTonesMap[tone] = i + 1
}
}
for k, v := range resource {
i, err := strconv.ParseInt(k, 16, 32)
if err != nil {
continue
}
pinyinMap[rune(i)] = v
}
initialized = true
})
py := &pinyin{
origin: origin,
split: " ",
mode: WithoutTone,
}
return py.Split("").Convert()
}
func (py *pinyin) Split(s string) *pinyin {
py.split = s
return py
}
func (py *pinyin) Mode(mode Mode) *pinyin {
py.mode = mode
return py
}
func (py *pinyin) Convert() (string, error) {
if !initialized {
return "", ErrInitialize
}
sr := []rune(py.origin)
words := make([]string, 0)
var temp string
for i, s := range sr {
_, ok := pinyinMap[s]
if !ok {
// 非中文处理
temp += string(s)
if i == len(sr)-1 {
words = append(words, temp)
}
continue
}
word, err := getPinyin(s, py.mode)
if err != nil {
return "", err
}
if len(temp) > 0 {
words = append(words, temp)
temp = ""
}
if len(word) > 0 {
words = append(words, word)
}
}
result := strings.Join(words, py.split)
result = strings.Replace(result, " ", " ", -1)
result = strings.Replace(result, " ", " ", -1)
return result, nil
}
func getPinyin(hanzi rune, mode Mode) (string, error) {
if !initialized {
return "", ErrInitialize
}
switch mode {
case Tone:
return getTone(hanzi), nil
case InitialsInCapitals:
return getInitialsInCapitals(hanzi), nil
default:
return getDefault(hanzi), nil
}
}
func getTone(hanzi rune) string {
return pinyinMap[hanzi]
}
func getDefault(hanzi rune) string {
tone := getTone(hanzi)
if tone == "" {
return tone
}
output := make([]rune, utf8.RuneCountInString(tone))
count := 0
for _, t := range tone {
neutral, found := tonesMap[t]
if found {
output[count] = neutral
} else {
output[count] = t
}
count++
}
return string(output)
}
func getInitialsInCapitals(hanzi rune) string {
def := getDefault(hanzi)
if def == "" {
return def
}
sr := []rune(def)
if sr[0] > 32 {
sr[0] = sr[0] - 32
}
return string(sr)
}
go-utils/readme.md (2.7 KiB)
常用工具包
- https://github.com/samber/lo
加密相关
- 生成 AES 密钥
- 生成 AES 密钥和 IV
- 计算文件md5(支持超大文件)
- 计算MD5大写、计算Md5小写
- SHA1、SHA256、HMacMd5、HMacSha1、SHAWithRSA
- Base64Encode、Base64Decode
- SHAWithRSA
- HideEmail
- IsEmail
大数据计算
- bigint 计算
Goroutine
- NewErrorGroup(context.TODO(), maxWorkers) 并发可取消的协程组
- AsyncFunc
- AsyncFuncPanic
- AsyncFuncGroup
- AsyncFuncGroupPanic
- MeasureExecutionTime
- IsContextDone
Utils
- func CheckSign(secret string, linkSignTimeout int64, ts int64, sign string) bool // 常用签名验证, sign md5 小写
- func CheckSignSha1(secret, nonce string, linkSignTimeout int64, ts int64, sign string) bool {
- func IsInArrayT any bool // 元素都转换成字符串比较
- func IsInArrayXT any bool
- func IsInArrayXXT any bool
- func IfNotT any T // 通用三目运算
- func IfString(isTrue bool, a, b string) string
- func IfInt(isTrue bool, a, b int) int
- func IfFloat32(isTrue bool, a, b float32) float32
- func IfFloat64(isTrue bool, a, b float64) float64
- func ReverseArray(arr []*interface{})
- func PadStart(str, pad string, length int) string
- func MinInt64(a, b int64) int64
- func MinInt(a, b int) int
- func MaxInt64(a, b int64) int64
- func MaxInt(a, b int) int
- func GenValidateCode(width int) string {//随机数
- func SplitStringArray(arr []string, size int) (list [][]string)
- func SplitIntArray(arr []int, size int) (list [][]int)
- func SplitInt64Array(arr []int64, size int) (list [][]int64)
- func SplitArray(arr []interface{}, size int) (list [][]interface{})
- func GetFieldValue(config interface{}, fieldName string) (interface{}, error) {// 通过反射获取结构体字段的值
- func CopyPropertiesT any // 复制对象
- func GetRuntimeStack() string // 获取运行堆栈
- func GenericSort // 通用排序
- func InsertionSort // 通用插入排序
- func FillMissingNumbers 填充缺失数字
- func ConvertAmPmHourTo24HourFormat Am Pm 时间转换比
password
- func BcryptHash(password string) string // BcryptHash 使用 bcrypt 对密码进行加密
- func BcryptCheck(password, hash string) bool // BcryptCheck 对比明文密码和数据库的哈希值
- func ValidPassword(str string) (msg string, matched bool) { //至少一位数字、大小字母,且长度6-20位
- func ValidPasswordV2(str string) (msg string, matched bool) { //至少一位数字、大小字母和特殊字符,且长度6-20位
time 时间
- time.go
- timex.go
xml
- xml.go
go-utils/safeslice.go (1.3 KiB)
package goutils
import (
golock "github.com/gif-gif/go.io/go-lock"
)
// 读写锁 + 顺序获取(循环)
//
// 如: s := []string{"s1", "s2", "s3"}, 无论并发怎么读取,顺序为:s1,s2,s3,s1,s2,s3,s1,s2,s...
type SafeSlice[T comparable] struct {
lock *golock.GoLock
currentKey T
data []T
}
func NewSafeSlice[T comparable]() *SafeSlice[T] {
return &SafeSlice[T]{
lock: &golock.GoLock{
//MuteRW: *new(sync.RWMutex),
},
}
}
func (m *SafeSlice[T]) Set(str T) {
m.lock.Lock()
defer m.lock.Unlock()
m.data = append(m.data, str)
m.data = m.removeDuplicates(m.data)
}
func (m *SafeSlice[T]) Sets(data []T) {
m.lock.Lock()
defer m.lock.Unlock()
if m.data == nil {
m.data = data
} else {
m.data = append(m.data, data...)
m.data = m.removeDuplicates(m.data)
}
}
func (m *SafeSlice[T]) removeDuplicates(s []T) []T {
seen := make(map[interface{}]struct{})
result := make([]T, 0, len(s))
for _, value := range s {
if _, ok := seen[value]; !ok {
seen[value] = struct{}{}
result = append(result, value)
}
}
return result
}
func (m *SafeSlice[T]) Get() T {
m.lock.Lock()
defer m.lock.Unlock()
count := len(m.data) - 1
key := m.data[0]
for k, v := range m.data {
if v == m.currentKey {
if k < count {
key = m.data[k+1]
break
}
}
}
m.currentKey = key
return key
}
go-utils/string.go (1.5 KiB)
package goutils
import (
"bytes"
"golang.org/x/text/encoding/simplifiedchinese"
"strings"
"unicode"
"unsafe"
)
// 多字符切割,默认支持逗号,分号,\n
func Split(s string, rs ...rune) []string {
return strings.FieldsFunc(s, func(r rune) bool {
for _, rr := range rs {
if rr == r {
return true
}
}
return r == ',' || r == ',' || r == ';' || r == ';' || r == '\n'
})
}
// 驼峰转下划线
func Camel2Case(str string) string {
var bf bytes.Buffer
for i, r := range str {
if !unicode.IsUpper(r) {
bf.WriteRune(r)
continue
}
if i > 0 {
bf.WriteString("_")
}
bf.WriteRune(unicode.ToLower(r))
}
return bf.String()
}
// 下划线转驼峰
func Case2Camel(str string) string {
str = strings.Replace(str, "_", " ", -1)
str = strings.Title(str)
return strings.Replace(str, " ", "", -1)
}
// 如果只需要转换单个字符串为 GBK
func UTF8ToGBK(text string) (string, error) {
encoder := simplifiedchinese.GBK.NewEncoder()
gbkBytes, err := encoder.Bytes([]byte(text))
if err != nil {
return "", err
}
return string(gbkBytes), nil
}
// GBK 转 UTF8
func GBKToUTF8(text string) (string, error) {
decoder := simplifiedchinese.GBK.NewDecoder()
bytes, err := decoder.Bytes([]byte(text))
if err != nil {
return "", err
}
return string(bytes), nil
}
// For Go 1.20 and higher
func StringToBytes(s string) []byte {
return unsafe.Slice(unsafe.StringData(s), len(s))
}
func BytesToString(b []byte) string {
return unsafe.String(unsafe.SliceData(b), len(b))
}
go-utils/time.go (3.0 KiB)
package goutils
import "time"
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 当前时区相关日期函数
func DateTime(format string) string {
return time.Now().Format(format)
}
func Today() string {
return DateTime(time.DateOnly)
}
func Now() string {
return DateTime(time.DateTime)
}
func NextDate(d int) string {
return time.Now().AddDate(0, 0, d).Format(time.DateOnly)
}
func Ts2Date(ts int64) string {
return time.Unix(ts, 0).Format(time.DateOnly)
}
func Ts2DateTime(ts int64) string {
return time.Unix(ts, 0).Format(time.DateTime)
}
func Date2Ts(date string) int64 {
ti, _ := time.ParseInLocation(time.DateOnly, date, time.Local)
return ti.Unix()
}
func DateTime2Ts(dateTime string) int64 {
return DateTime2TsLocalFormat(dateTime, time.Local, time.DateTime)
}
//--------------本地化时间--------------
//没有format 时日志格式为 time.DateOnly 和 time.DateTime
// 当前时间通用 格式用法
func DateTimeLocal(format string, timeOffsetSec int) string {
return Ts2DateTimeLocalFormat(time.Now().Unix(), timeOffsetSec, format)
}
// NowDate
func TodayLocal(timeOffsetSec int) string {
return Ts2DateLocal(time.Now().Unix(), timeOffsetSec)
}
// NowDateTime
func NowLocal(timeOffsetSec int) string {
return Ts2DateTimeLocal(time.Now().Unix(), timeOffsetSec)
}
// NextDate
func NextDateLocal(d int, timeOffsetSec int) string {
location := time.FixedZone("OffsetZone", timeOffsetSec)
t := time.Unix(time.Now().Unix(), 0)
t.In(location)
return Ts2DateTimeLocalFormat(t.AddDate(0, 0, d).Unix(), timeOffsetSec, time.DateOnly)
}
// 本地化日期 给定时间戳(以秒为单位)
func Ts2DateLocal(ts int64, timeOffsetSec int) string {
return Ts2DateTimeLocalFormat(ts, timeOffsetSec, time.DateOnly)
}
// 本地化日期时间 给定时间戳(以秒为单位)
func Ts2DateTimeLocal(ts int64, timeOffsetSec int) string {
return Ts2DateTimeLocalFormat(ts, timeOffsetSec, time.DateTime)
}
func Date2TsLocal(date string, location *time.Location) int64 {
return DateTime2TsLocalFormat(date, location, time.DateOnly)
}
func DateTime2TsLocal(dateTime string, location *time.Location) int64 {
return DateTime2TsLocalFormat(dateTime, location, time.DateTime)
}
func DateTime2TsLocalFormat(dateTime string, location *time.Location, format string) int64 {
ti, _ := time.ParseInLocation(format, dateTime, location)
return ti.Unix()
}
// 将时间对象转换为特定时区的时间
func Ts2DateTimeLocalFormat(ts int64, timeOffsetSec int, format string) string {
t := time.Unix(ts, 0)
location := time.FixedZone("OffsetZone", timeOffsetSec)
// 将时间对象转换为特定时区的时间
localTime := t.In(location)
return localTime.Format(format)
}
go-utils/timex.go (8.9 KiB)
package goutils
import (
"errors"
"fmt"
"math"
"regexp"
"strings"
"time"
)
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
// 这个包即将废弃,改用 gotime 包
var (
TimeLayout string = "2006-01-02 15:04:05"
DateLayout = "20060102"
TimeFormat = map[string]string{
"Y-m-d H:i:s": "2006-01-02 15:04:05",
"Y-m-d": "2006-01-02",
"Ymd": "20060102",
"H:i:s": "15:04:05",
"Y": "2006",
"m": "01",
"d": "02",
}
)
// GetTimeNow 获取当前时间GetTimeNow(),用于测试时的时间修改
func GetTimeNow() time.Time {
//redisConf := redis.RedisKeyConf{
// RedisConf: redis.RedisConf{
// Host: "122.228.113.235:17006",
// Type: "",
// Pass: "xiaozi527sport",
// TLS: false,
// },
//}
//redisClient := redisConf.NewRedis()
//val, err := redisClient.Get("test:now:gap:seconds")
//if err != nil {
// fmt.Errorf("GetTimeNow test:now:gap error:%s", err.Error())
// return time.Now()
//}
//gap, _ := strconv.ParseInt(val, 10, 64)
//return time.Now().Add(time.Second * time.Duration(gap))
return time.Now()
}
func Ts2Time(t int64) time.Time {
return time.Unix(t, 0)
}
func BeijingTimeLocation() *time.Location {
loc, _ := time.LoadLocation("Asia/Shanghai")
return loc
}
func GetChinaTomorrowAMSeconds(isBeijing bool) int64 {
now := GetTimeNow()
if isBeijing {
loc, _ := time.LoadLocation("Asia/Shanghai")
t, _ := time.ParseInLocation("2006-01-02", now.AddDate(0, 0, 1).Format("2006-01-02"), loc)
secondsF := t.Sub(GetTimeNow()).Seconds()
return int64(secondsF)
} else {
secondsF := now.Sub(GetTimeNow()).Seconds()
return int64(secondsF)
}
}
func GetLocalTomorrowAMSeconds() int64 {
now := GetTimeNow()
t, _ := time.ParseInLocation("2006-01-02", now.AddDate(0, 0, 1).Format("2006-01-02"), time.Local)
secondsF := t.Sub(GetTimeNow()).Seconds()
return int64(secondsF)
}
func GetTodayZero() time.Time {
t := GetTimeNow()
zero := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
return zero
}
func GetZero(targetTime time.Time) time.Time {
zero := time.Date(targetTime.Year(), targetTime.Month(), targetTime.Day(), 0, 0, 0, 0, targetTime.Location())
return zero
}
// ParseTime 解析时间,"2021-03-17 00:00:00"
func ParseTime(timeStr string) (datetime time.Time) {
datetime, _ = time.ParseInLocation(TimeLayout, timeStr, time.Local)
return
}
//eg:20210812170000
func ParseTimeString(timeStr string) (datetime time.Time) {
datetime, _ = time.ParseInLocation("20060102150405", timeStr, time.Local)
return
}
func GetDateInterval(t1, t2 time.Time) int {
t1 = time.Date(t1.Year(), t1.Month(), t1.Day(), 0, 0, 0, 0, time.Local)
t2 = time.Date(t2.Year(), t2.Month(), t2.Day(), 0, 0, 0, 0, time.Local)
interval := int(math.Abs(t1.Sub(t2).Hours())/24) + 1
return interval
}
// SinceDays 获取过去的天数,dateString格式20060102
func SinceDays(dateString string) (int64, error) {
targetTime, err := time.ParseInLocation("20060102", dateString, time.Local)
if err != nil {
return 0, err
} else {
days := int64(math.Ceil(time.Since(targetTime).Hours() / 24))
if days == 0 {
days = 1
}
return days, nil
}
}
func IsSameDay(t1, t2 time.Time) bool {
year1, month1, day1 := t1.Date()
year2, month2, day2 := t2.Date()
return day1 == day2 && month1 == month2 && year1 == year2
}
// LastHourStartAndEnd 上一个小时的开始和结束时间戳
func LastHourStartAndEnd(isBeijing bool) (int, int64, int64) {
now1 := time.Now()
var now time.Time
if isBeijing {
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
fmt.Println("Error loading location:", err)
return 0, 0, 0
}
now = now1.In(location)
} else {
now = now1
}
startOfLastHour := time.Date(now.Year(), now.Month(), now.Day(), now.Hour()-1, 0, 0, 0, now.Location())
endOfLastHour := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location()).Add(-time.Nanosecond)
startOfLastHourUnix := startOfLastHour.Unix()
endOfLastHourUnix := endOfLastHour.Unix()
return startOfLastHour.Hour(), startOfLastHourUnix, endOfLastHourUnix
}
func CurrentHourStartAndEnd() (int, int64, int64) {
now := time.Now()
startOfHour := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
endOfHour := startOfHour.Add(time.Hour - time.Second)
startOfHourTimestamp := startOfHour.Unix()
endOfHourTimestamp := endOfHour.Unix()
return startOfHour.Hour(), startOfHourTimestamp, endOfHourTimestamp
}
func GetNowDateForLocation(isBeijing bool) string {
tm := time.Now()
var tmInLocation time.Time
if isBeijing {
// 加载时区
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
fmt.Println("Error loading location:", err)
return ""
}
// 将时间转换为指定时区
tmInLocation = tm.In(location)
} else {
tmInLocation = tm
}
// 将时间格式化为日期字符串
dateStr := tmInLocation.Format(time.DateOnly)
return dateStr
}
func ToDate(ts int64, isBeijing bool) string {
tm := time.Unix(ts, 0)
var tmInLocation time.Time
if isBeijing {
// 加载时区
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
fmt.Println("Error loading location:", err)
return ""
}
// 将时间转换为指定时区
tmInLocation = tm.In(location)
} else {
tmInLocation = tm
}
// 将时间格式化为日期字符串
dateStr := tmInLocation.Format(time.DateOnly)
return dateStr
}
func ToDateTime(ts int64, isBeijing bool) string {
tm := time.Unix(ts, 0)
var tmInLocation time.Time
if isBeijing {
// 加载时区
location, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
fmt.Println("Error loading location:", err)
return ""
}
// 将时间转换为指定时区
tmInLocation = tm.In(location)
} else {
tmInLocation = tm
}
// 将时间格式化为日期字符串
dateStr := tmInLocation.Format(time.DateTime)
return dateStr
}
// 返回相差天数
func TimeRangeDay(stTime int64, endTIme int64) int {
startTime := time.Unix(stTime, 0)
endTime := time.Unix(endTIme, 0)
durationDays := int(endTime.Sub(startTime).Hours()/24 + 1)
return durationDays
}
func TimeRangeDates(startDate string, endDate string) []string {
//startDate := "2021-01-01"
//endDate := "2021-01-10"
dates := []string{}
start, _ := time.Parse("2006-01-02", startDate)
end, _ := time.Parse("2006-01-02", endDate)
for d := start; !d.After(end); d = d.AddDate(0, 0, 1) {
fmt.Println(d.Format("2006-01-02"))
dates = append(dates, d.Format("2006-01-02"))
}
return dates
}
func DatesForRangeTs(startTimestamp int64, endTimestamp int64, format string) []string {
if format == "" {
format = "2006-01-02"
}
// 转换为time.Time对象
startTime := time.Unix(startTimestamp, 0)
endTime := time.Unix(endTimestamp, 0)
// 创建一个当前时间的副本
currentTime := startTime
dates := []string{}
// 循环直到达到结束时间
for currentTime.Before(endTime) {
// 格式化输出日期
dates = append(dates, currentTime.Format(format))
// 增加一天
currentTime = currentTime.AddDate(0, 0, 1)
}
// 输出最后一天(如果最后一天的时间戳在当天内)
if currentTime.Format(format) == endTime.Format(format) {
dates = append(dates, currentTime.Format(format))
}
return dates
}
// convertTo24HourFormat 将时间字符串转换为 0-23 小时制
// timeStr: 时间字符串,例如 "3:04 PM"
// 3:04 PM-->15:04 ,必须是这两个格式之一
func ConvertAmPmHourTo24HourFormat(timeStr string, layout24Hour string) (string, error) {
// 定义时间解析格式
const layout12Hour = "3:04 PM"
if layout24Hour == "" {
layout24Hour = "15:04"
}
if !strings.HasPrefix(layout24Hour, "15") {
return "", errors.New("layout24Hour must start with 15 or 15:04")
}
// 解析时间字符串
t, err := time.Parse(layout12Hour, timeStr)
if err != nil {
return "", fmt.Errorf("failed to parse time: %v", err)
}
// 格式化为 24 小时制
return t.Format(layout24Hour), nil
}
// 格式为: 2021-08-06T07:00:00+0000 To time.Time
func ConvertToGMTTime(gmtTime string) (time.Time, error) {
t, err := time.Parse("2006-01-02T15:04:05-0700", gmtTime)
return t, err
}
// 判断是不是 2024-11-22 4 格式的时间串
func IsValidDateTime(str string) bool {
// 定义日期时间格式的正则表达式,忽略多余空格
re := regexp.MustCompile(`^\d{4}-\d{1,2}-\d{1,2}\s+\d{1,2}$`)
// 使用正则表达式匹配字符串
isValidFormat := re.MatchString(strings.TrimSpace(str))
return isValidFormat
}
func IsValidDate(str string) bool {
// 定义日期时间格式的正则表达式,忽略多余空格
re := regexp.MustCompile(`^\d{4}-\d{1,2}-\d{1,2}$`)
// 使用正则表达式匹配字符串
isValidFormat := re.MatchString(strings.TrimSpace(str))
return isValidFormat
}
go-utils/utils.go (7.3 KiB)
package goutils
import (
"fmt"
"github.com/gogf/gf/util/gconv"
"github.com/samber/lo"
"golang.org/x/crypto/bcrypt"
"math"
"math/rand"
"net"
"reflect"
"regexp"
"runtime"
"slices"
"sort"
"strings"
"time"
)
// BcryptHash 使用 bcrypt 对密码进行加密
func BcryptHash(password string) string {
bytes, _ := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
return string(bytes)
}
// BcryptCheck 对比明文密码和数据库的哈希值
func BcryptCheck(password, hash string) bool {
err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password))
return err == nil
}
// 常用签名验证, sign md5 小写
func CheckSign(secret string, linkSignTimeout int64, ts int64, sign string) bool {
if linkSignTimeout == 0 {
linkSignTimeout = 20
}
tsStep := time.Now().Unix() - ts
if math.Abs(gconv.Float64(tsStep)) > gconv.Float64(linkSignTimeout) { //连接失效
return false
}
serverSign := Md5([]byte(gconv.String(ts) + secret))
return serverSign == sign
}
// 元素都转换成字符串比较
func IsInArray[T comparable](arr []T, target T) bool {
return slices.Contains(arr, target)
}
// 条件满足任意元素 exists func(target T) bool 返回true时返回true
//
// 适合判断数组中存储复杂对象,判断条件定义情况
//
// 用以下代替
//
// slices.ContainsFunc(arr, func(t T) bool {
//
// })
func IsInArrayX[T any](arr []T, exists func(target T) bool) bool {
for _, t := range arr {
if exists(t) {
return true
}
}
return false
}
// 条件满足任意元素 exists func(target *T) bool 返回true时返回true
//
// 适合判断数组中存储复杂对象,判断条件定义情况,数组元素是指针类型时用
//
// 用以下代替
//
// slices.ContainsFunc(arr, func(t T) bool {
//
// })
func IsInArrayXX[T any](arr []*T, exists func(target *T) bool) bool {
for _, t := range arr {
if exists(t) {
return true
}
}
return false
}
// 通用三目运算
func IfNot[T any](isTrue bool, a, b T) T {
if isTrue {
return a
}
return b
}
func IfString(isTrue bool, a, b string) string {
if isTrue {
return a
}
return b
}
func IfInt(isTrue bool, a, b int) int {
if isTrue {
return a
}
return b
}
func IfFloat32(isTrue bool, a, b float32) float32 {
if isTrue {
return a
}
return b
}
func IfFloat64(isTrue bool, a, b float64) float64 {
if isTrue {
return a
}
return b
}
func ReverseArray(arr []*interface{}) {
for i, j := 0, len(arr)-1; i <= j; i, j = i+1, j-1 {
arr[i], arr[j] = arr[j], arr[i]
}
}
func PadStart(str, pad string, length int) string {
if len(str) >= length {
return str
}
return strings.Repeat(pad, length-len(str)) + str
}
func MinInt64(a, b int64) int64 {
return gconv.Int64(math.Min(gconv.Float64(a), gconv.Float64(b)))
}
func MinInt(a, b int) int {
return gconv.Int(math.Min(gconv.Float64(a), gconv.Float64(b)))
}
func MaxInt64(a, b int64) int64 {
return gconv.Int64(math.Max(gconv.Float64(a), gconv.Float64(b)))
}
func MaxInt(a, b int) int {
return gconv.Int(math.Max(gconv.Float64(a), gconv.Float64(b)))
}
func GenValidateCode(width int) string {
numeric := [10]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
r := len(numeric)
rand.Seed(time.Now().UnixNano())
var sb strings.Builder
for i := 0; i < width; i++ {
fmt.Fprintf(&sb, "%d", numeric[rand.Intn(r)])
}
return sb.String()
}
// 通过反射获取结构体字段的值
func GetFieldValue(config interface{}, fieldName string) (interface{}, error) {
v := reflect.ValueOf(config)
// 确保传入的是一个指针
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
// 确保传入的是结构体
if v.Kind() != reflect.Struct {
return nil, fmt.Errorf("expected a struct, but got %s", v.Kind())
}
// 获取字段值
field := v.FieldByName(fieldName)
if !field.IsValid() {
return nil, fmt.Errorf("no such field: %s in struct", fieldName)
}
return field.Interface(), nil
}
// 使用反射复制A结构到B结构,前提是两个结构体字段数量和类型完全相同
// 如:使用反射将 VO 转换为 DTO
// 反射(reflect)虽爽,但很贵,性能会有损失
func CopyProperties[T any](target interface{}) T {
var t T
voValue := reflect.ValueOf(target)
dtoValue := reflect.New(reflect.TypeOf(t)).Elem()
for i := 0; i < voValue.NumField(); i++ {
dtoField := dtoValue.Field(i)
voField := voValue.Field(i)
dtoField.Set(voField)
}
return dtoValue.Interface().(T)
}
func GetRuntimeStack() string {
var buf [4096]byte
n := runtime.Stack(buf[:], false)
return string(buf[:n])
}
// 插入排序函数,使用泛型指定元素类型
func InsertionSort[T any](arr []T, less func(T, T) bool) {
n := len(arr)
for i := 1; i < n; i++ {
key := arr[i]
j := i - 1
// 将比key大的元素向后移动一位
for j >= 0 && less(arr[j], key) == false {
arr[j+1] = arr[j]
j--
}
// 插入关键元素到正确的位置
arr[j+1] = key
}
}
// 定义一个泛型排序函数
func GenericSort[T any](arr []T, less func(T, T) bool) {
sort.Slice(arr, func(i, j int) bool {
return less(arr[i], arr[j])
})
}
// 把缺失的数字填充到数组中
func FillMissingNumbers(nums []int64, max int64) []int64 {
// 创建一个新的切片来存储结果
var result []int64
// 从 1 开始
current := int64(1)
// 遍历给定的数字
for _, num := range nums {
// 填充中间缺失的数字
for current < num {
result = append(result, current)
current++
}
// 添加当前数字
result = append(result, num)
current = num + 1 // 更新当前数字到下一个
}
// 如果还有剩余的数字,继续填充
for i := current; i <= max; i++ { // 假设我们想填充到 20
result = append(result, i)
}
return result
}
func GetPageCount(total int64, pageSize int64) (totalPages int64) {
return int64(math.Ceil(float64(total) / float64(pageSize)))
}
// 下一页
func AfterPage(page int64, pageCount int64) int64 {
if page <= 0 {
page = 1
}
after := page + 1
if after > pageCount {
after = -1
}
return after
}
func AfterPageValue(page int64, total int64, pageSize int64) int64 {
if page <= 0 {
page = 1
}
pageCount := GetPageCount(total, pageSize)
after := page + 1
if after > pageCount {
after = -1
}
return after
}
func BeforePage(page int64) int64 {
before := page - 1
if before <= 0 {
before = -1
}
return before
}
// 是不是数字
func IsNumeric(str string) bool {
re := regexp.MustCompile(`^[0-9]+(\.[0-9]+)?$`)
// 使用正则表达式匹配字符串
ok := re.MatchString(str)
return ok
}
func IsInt(str string) bool {
re := regexp.MustCompile("^[0-9]+$")
// 使用正则表达式匹配字符串
ok := re.MatchString(str)
return ok
}
func Sum(list []int) int {
return lo.Sum(list)
}
// 两个数组是否相等,判断长度一样的两个数组 元素是否完全相同,顺序可以不同
func IsEqualArray[T comparable](arr1 []T, arr2 []T) bool {
if len(arr1) != len(arr2) {
return false
}
for _, v := range arr1 {
if !lo.Contains(arr2, v) {
return false
}
}
return true
}
// 方法1: 使用 net.ParseIP(推荐)
func IsIPv4(str string) bool {
ip := net.ParseIP(str)
if ip == nil {
return false
}
// 检查是否是 IPv4
return ip.To4() != nil
}
// 方法2: 正则表达式(简单版)
func IsIPv4Regex(str string) bool {
// 简单的 IPv4 正则(不完全准确)
pattern := `^(\d{1,3}\.){3}\d{1,3}$`
matched, _ := regexp.MatchString(pattern, str)
return matched
}
go-utils/validation.go (1.0 KiB)
package goutils
import "regexp"
func ValidPassword(str string) (msg string, matched bool) {
msg = "至少一位数字、大小字母,且长度6-20位"
matched, _ = regexp.MatchString("[0-9]+", str)
if !matched {
return
}
matched, _ = regexp.MatchString("[a-z]+", str)
if !matched {
return
}
matched, _ = regexp.MatchString("[A-Z]+", str)
if !matched {
return
}
if l := len(str); l < 6 || l > 20 {
matched = false
return
}
msg = ""
matched = true
return
}
func ValidPasswordV2(str string) (msg string, matched bool) {
msg = "至少一位数字、大小字母和特殊字符,且长度6-20位"
matched, _ = regexp.MatchString("[0-9]+", str)
if !matched {
return
}
matched, _ = regexp.MatchString("[a-z]+", str)
if !matched {
return
}
matched, _ = regexp.MatchString("[A-Z]+", str)
if !matched {
return
}
matched, _ = regexp.MatchString("[^0-9a-zA-Z]+", str)
if !matched {
return
}
if l := len(str); l < 6 || l > 20 {
matched = false
return
}
msg = ""
matched = true
return
}
go-utils/xml.go (740 B)
package goutils
import (
"encoding/xml"
"io"
)
type StringMap map[string]string
type xmlMapEntry struct {
XMLName xml.Name
Value string `xml:",chardata"`
}
func (m StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if len(m) == 0 {
return nil
}
err := e.EncodeToken(start)
if err != nil {
return err
}
for k, v := range m {
e.Encode(xmlMapEntry{XMLName: xml.Name{Local: k}, Value: v})
}
return e.EncodeToken(start.End())
}
func (m *StringMap) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) error {
*m = StringMap{}
for {
var e xmlMapEntry
err := d.Decode(&e)
if err == io.EOF {
break
} else if err != nil {
return err
}
(*m)[e.XMLName.Local] = e.Value
}
return nil
}
go-xlsx/csv_reader.go (4.9 KiB)
package goxlsx
import (
"encoding/csv"
"fmt"
gofile "github.com/gif-gif/go.io/go-file"
"golang.org/x/text/encoding"
"golang.org/x/text/transform"
"io"
"os"
)
type CsvReader struct {
FilePath string
Comma rune //csv 列分割符
file *os.File
}
// comma 默认传 ','
func NewCsvReader(csvFile string, comma rune) (*CsvReader, error) {
e, err := gofile.Exist(csvFile)
if err != nil {
return nil, err
}
if !e {
return nil, fmt.Errorf("file not exist")
}
return &CsvReader{
FilePath: csvFile,
Comma: comma,
}, nil
}
func (c *CsvReader) getReader(encoding encoding.Encoding) (*csv.Reader, error) {
// 打开 CSV 文件
file, err := os.Open(c.FilePath)
if err != nil {
return nil, fmt.Errorf("failed to open file: %v", err)
}
c.file = file
reader := csv.NewReader(transform.NewReader(file, encoding.NewDecoder()))
reader.Comma = c.Comma
return reader, nil
}
func (c *CsvReader) ReadGBKAll() ([][]string, error) {
// 打开 CSV 文件
return c.ReadAll(GBK)
}
func (c *CsvReader) ReadUTF8All() ([][]string, error) {
// 打开 CSV 文件
return c.ReadAll(UTF8)
}
func (c *CsvReader) ReadUTF16All() ([][]string, error) {
return c.ReadAll(UTF16)
}
func (c *CsvReader) ReadUTF8Line(lineDataFunc func(record []string) error) error {
return c.ReadLine(UTF8, lineDataFunc)
}
func (c *CsvReader) ReadUTF16Line(lineDataFunc func(record []string) error) error {
// 创建 UTF-16 解码器
return c.ReadLine(UTF16, lineDataFunc)
}
func (c *CsvReader) ReadGBKLine(lineDataFunc func(record []string) error) error {
return c.ReadLine(GBK, lineDataFunc)
}
func (c *CsvReader) ReadAll(encoding encoding.Encoding) ([][]string, error) {
// 打开 CSV 文件
// 创建 UTF-16 解码器
reader, err := c.getReader(encoding)
if err != nil {
return nil, err
}
defer c.Close()
// 读取所有记录
records, err := reader.ReadAll()
if err != nil {
return nil, fmt.Errorf("failed to read file: %v", err)
}
return records, nil
}
func (c *CsvReader) ReadLine(encoding encoding.Encoding, lineDataFunc func(record []string) error) error {
// 打开 CSV 文件
reader, err := c.getReader(encoding)
if err != nil {
return err
}
defer c.Close()
// 按行读取文件
for {
record, err := reader.Read()
if err != nil {
if err == io.EOF {
break // 文件读取完毕
}
return fmt.Errorf("failed to read line: %v", err)
}
// 处理每一行数据
e := lineDataFunc(record)
if e != nil {
return e
}
}
return nil
}
// 第一行作为字段名称,后续行数据转换为json数据,一行回调一个json数据
func (c *CsvReader) ReadLineJson(encoding encoding.Encoding, lineDataFunc func(record map[string]string) error) error {
// 打开 CSV 文件
reader, err := c.getReader(encoding)
if err != nil {
return err
}
defer c.Close()
// 获取标题行
// 创建一个切片来保存 JSON 对象
// 遍历每一行(从第二行开始,因为第一行是标题)
record, err := reader.Read()
if err != nil {
return err
}
headers := record
// 遍历每一行(从第二行开始,因为第一行是标题)
// 按行读取文件
for {
record, err := reader.Read()
if err != nil {
if err == io.EOF {
break // 文件读取完毕
}
return fmt.Errorf("failed to read line: %v", err)
}
// 处理每一行数据
row := make(map[string]string)
for i, value := range record {
row[headers[i]] = value
}
e := lineDataFunc(row)
if e != nil {
return e
}
}
return nil
}
// 第一行作为字段名称,后续行数据转换为json数据一次性返回所有数据
func (c *CsvReader) ReadAllJson(encoding encoding.Encoding) ([]map[string]string, error) {
// 打开 CSV 文件
reader, err := c.getReader(encoding)
if err != nil {
return nil, err
}
defer c.Close()
// 读取 CSV 文件的所有内容
records, err := reader.ReadAll()
if err != nil {
return nil, fmt.Errorf("failed to read CSV file: %v", err)
}
// 检查 CSV 文件是否为空
if len(records) < 2 {
return nil, fmt.Errorf("CSV file does not contain enough data")
}
// 获取标题行
headers := records[0]
// 创建一个切片来保存 JSON 对象
var jsonData []map[string]string
// 遍历每一行(从第二行开始,因为第一行是标题)
for _, record := range records[1:] {
// 创建一个 map 来保存每一行的数据
row := make(map[string]string)
for i, value := range record {
row[headers[i]] = value
}
// 将 map 添加到 jsonData 切片中
jsonData = append(jsonData, row)
}
return jsonData, nil
}
func (c *CsvReader) ReadTitles(encoding encoding.Encoding) ([]string, error) {
// 打开 CSV 文件
reader, err := c.getReader(encoding)
if err != nil {
return nil, err
}
defer c.Close()
// 获取标题行
// 第一行是标题视为 标题
record, err := reader.Read()
if err != nil {
return nil, err
}
headers := record
return headers, nil
}
func (c *CsvReader) Close() {
if c.file != nil {
c.file.Close()
}
}
go-xlsx/csv_writer.go (5.6 KiB)
package goxlsx
import (
"encoding/csv"
"fmt"
"github.com/gin-gonic/gin"
"golang.org/x/text/encoding"
"golang.org/x/text/transform"
"io"
"net/http"
"os"
"path"
)
type CsvWriter struct {
FilePath string
Comma rune //csv 列分割符
//只有逐行写入时才保留这个状态
file *os.File
writer *csv.Writer //for local file writer
titles []string //for http writer
rows [][]string //for http writer
encoding.Encoding
}
func NewCsvWriter(csvFile string, comma rune, encoding ...encoding.Encoding) (*CsvWriter, error) {
encode := UTF8
if len(encoding) > 0 {
encode = encoding[0]
}
return &CsvWriter{
FilePath: csvFile,
Comma: comma,
Encoding: encode,
}, nil
}
// 不需要创建本地文件, 只写入数据。如:网络IO
func NewCsvWriterNoneFile(comma rune) (*CsvWriter, error) {
return NewCsvWriter("", comma)
}
func (c *CsvWriter) getWriter() (*csv.Writer, error) {
// 打开 CSV 文件
// Open file in append mode
dirname := path.Dir(c.FilePath)
if _, err := os.Stat(dirname); err != nil {
os.MkdirAll(dirname, 0755)
}
file, err := os.OpenFile(c.FilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nil, fmt.Errorf("failed to open file: %v", err)
}
err = c.WriteBom(file)
if err != nil {
file.Close()
return nil, err
}
c.file = file
// 创建一个 CSV 写入器
writer := csv.NewWriter(transform.NewWriter(file, c.Encoding.NewEncoder()))
writer.Comma = c.Comma // 使用分号作为分隔符
return writer, nil
}
// 一次性写入
func (c *CsvWriter) WriteData(records [][]string) error {
// 写入记录到 CSV 文件
dirname := path.Dir(c.FilePath)
if _, err := os.Stat(dirname); err != nil {
os.MkdirAll(dirname, 0755)
}
file, err := os.OpenFile(c.FilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return fmt.Errorf("failed to open file: %v", err)
}
defer file.Close()
err = c.WriteBom(file)
if err != nil {
return err
}
// 创建一个 CSV 写入器
writer := csv.NewWriter(transform.NewWriter(file, c.Encoding.NewEncoder()))
writer.Comma = c.Comma // 使用分号作为分隔符
defer writer.Flush() // 确保在函数结束时刷新写入器
// 写入记录到 CSV 文件
if err := writer.Write(c.titles); err != nil { //如果有titles 则自动写入
return err
}
err = writer.WriteAll(records)
if err != nil {
return err
}
return nil
}
// AppendToCSV appends data to an existing CSV file
// 适合一次性追加多条数据,每次会打开文件,写入,关闭文件,效率低
func (c *CsvWriter) AppendToCSV(data [][]string) error {
// Create CSV writer
// 打开 CSV 文件
// Open file in append mode
file, err := os.OpenFile(c.FilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return fmt.Errorf("failed to open file: %v", err)
}
defer file.Close()
// 创建一个 CSV 写入器
writer := csv.NewWriter(transform.NewWriter(file, c.Encoding.NewEncoder()))
writer.Comma = c.Comma // 使用分号作为分隔符
defer writer.Flush()
// Write all data rows
for _, row := range data {
if err := writer.Write(row); err != nil {
return err
}
}
return nil
}
// 逐行写入, 需要手动调用 c.Close() 或者 end 为true 时才会关闭文件
func (c *CsvWriter) WriteLine(record []string, end ...bool) error {
if c.writer == nil { //不存在 则创建
w, err := c.getWriter()
if err != nil {
return err
}
c.writer = w
// 写入记录到 CSV 文件
if err := c.writer.Write(c.titles); err != nil { //如果有titles 则自动写入
return err
}
}
if end != nil && len(end) > 0 && end[0] {
defer c.Close()
}
// 写入记录到 CSV 文件
defer c.writer.Flush() // 确保在函数结束时刷新写入器
// 写入记录到 CSV 文件
if err := c.writer.Write(record); err != nil {
return err
}
return nil
}
// 关闭文件 c.file.Close()
func (c *CsvWriter) Close() {
if c.file != nil {
c.file.Close()
}
c.writer = nil
}
func (c *CsvWriter) SetTitles(titles []string) {
c.titles = titles
}
func (c *CsvWriter) getTitles() []string {
return c.titles
}
func (x *CsvWriter) AppendData(data []string) *CsvWriter {
x.rows = append(x.rows, data)
return x
}
func (x *CsvWriter) AppendRows(data [][]string) *CsvWriter {
for _, i := range data {
x.rows = append(x.rows, i)
}
return x
}
// 一次性把titles、rows输出到网络
func (x *CsvWriter) OutputForGin(ctx *gin.Context, filename string) (err error) {
ctx.Header("Content-Type", "application/octet-stream")
ctx.Header("Content-Disposition", "attachment; filename="+filename)
ctx.Header("Content-Transfer-Encoding", "binary")
ctx.Header("Expires", "0")
return x.Output(ctx.Writer)
}
// 一次性把titles、rows输出到网络
func (x *CsvWriter) OutputResponseWriter(w http.ResponseWriter, filename string) (err error) {
header := w.Header()
header.Set("Content-Type", "application/octet-stream")
header.Set("Content-Disposition", "attachment; filename="+filename)
header.Set("Content-Transfer-Encoding", "binary")
header.Set("Expires", "0")
return x.Output(w)
}
func (x *CsvWriter) Output(w io.Writer) (err error) {
err = x.WriteBom(w)
if err != nil {
return err
}
writer := csv.NewWriter(w)
defer writer.Flush()
if len(x.titles) > 0 {
if err := writer.Write(x.titles); err != nil {
return err
}
}
// 写入数据行
for _, row := range x.rows {
if err := writer.Write(row); err != nil {
return err
}
}
return nil
}
// 写入BOM
func (x *CsvWriter) WriteBom(w io.Writer) error {
if x.Encoding == UTF8 {
_, err := w.Write(BOM_UTF8)
if err != nil {
return err
}
} else if x.Encoding == UTF16 {
_, err := w.Write(BOM_UTF16)
if err != nil {
return err
}
}
return nil
}
go-xlsx/readme.md (109 B)
go xlxs
基于这个库封装
go get github.com/xuri/excelize/v2
- 目前只支持写一个sheet
go-xlsx/types.go (616 B)
package goxlsx
import (
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/encoding/unicode"
)
// 常用文件编码
// unicode.UTF8,
// unicode.UTF16(BigEndian, UseBOM),
// unicode.UTF16(BigEndian, IgnoreBOM),
// unicode.UTF16(LittleEndian, IgnoreBOM),
// 常用文件编码需要用
var (
UTF8 = unicode.UTF8
UTF8BOM = unicode.UTF8BOM
GBK = simplifiedchinese.GBK
//UTF16 有很多种 参考 unicode包
UTF16 = unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM)
)
// 常用文件编码写入bom头部
var (
BOM_UTF8 = []byte{0xEF, 0xBB, 0xBF}
BOM_UTF16 = []byte{0xFF, 0xFE}
)
go-xlsx/xlsx_reader.go (909 B)
package goxlsx
import (
"github.com/xuri/excelize/v2"
)
// 需要扩展按行读取文件
type XlsxRead struct {
FilePath string
}
func NewReader(xlsxFile string) (*XlsxRead, error) {
return &XlsxRead{
FilePath: xlsxFile,
}, nil
}
func (r *XlsxRead) ReadBySheet(sheet string, fn func(n int, row []string) error) error {
xlsx, err := excelize.OpenFile(r.FilePath)
if err != nil {
return err
}
defer xlsx.Close()
if sheet == "" {
sheet = xlsx.GetSheetName(0)
}
rows, err := xlsx.Rows(sheet)
if err != nil {
return err
}
defer rows.Close()
var n int
for rows.Next() {
n++
row, _ := rows.Columns()
if err = fn(n, row); err != nil {
return err
}
}
return nil
}
func (r *XlsxRead) Read(fn func(n int, row []string) error) error {
return r.ReadBySheet("", fn)
}
func (r *XlsxRead) ReadFile(file string, fn func(n int, row []string) error) error {
return r.Read(fn)
}
go-xlsx/xlsx_test.go (1.1 KiB)
package goxlsx
import (
golog "github.com/gif-gif/go.io/go-log"
"testing"
)
func TestXlsxWrite(t *testing.T) {
w := NewWriter()
w.SetSheetName("test")
w.titles = &[]string{"测试", "title2", "title3"}
var data [][]interface{}
for i := 0; i < 10; i++ {
data = append(data, []interface{}{"1", "2", "3"})
}
w.AppendRows(data)
err := w.Save2File("test.xlsx")
if err != nil {
golog.Error(err)
}
}
func TestCsvRead(t *testing.T) {
w, err := NewCsvReader("/Users/Jerry/Documents/my/test/data/detail.csv", ',')
if err != nil {
golog.Error(err)
return
}
err = w.ReadUTF16Line(func(record []string) error {
golog.WithTag("record").Info(record)
return nil
})
if err != nil {
golog.Error(err)
return
}
}
func TestCsvRead1(t *testing.T) {
w, err := NewCsvReader("/Users/Jerry/Documents/my/test/meta/meta.csv", ',')
if err != nil {
golog.Error(err)
return
}
line := 0
err = w.ReadLineJson(UTF8, func(record map[string]string) error {
golog.WithTag("record").Info(record)
line++
return nil
})
if err != nil {
golog.Error(err)
return
}
golog.Info(line)
}
go-xlsx/xlsx_writer.go (2.6 KiB)
package goxlsx
import (
"fmt"
"github.com/gin-gonic/gin"
"github.com/xuri/excelize/v2"
"golang.org/x/text/encoding"
"io"
"net/http"
)
// 扩展需要append to file 功能
type XlsxWrite struct {
fh *excelize.File
sheetName string
titles *[]string
rows []*[]interface{}
encoding.Encoding
}
func NewWriter(encoding ...encoding.Encoding) *XlsxWrite {
encode := UTF8
if len(encoding) > 0 {
encode = encoding[0]
}
return &XlsxWrite{
fh: excelize.NewFile(),
sheetName: "Sheet1",
Encoding: encode,
}
}
func (x *XlsxWrite) SetTitles(titles []string) *XlsxWrite {
x.titles = &titles
return x
}
func (x *XlsxWrite) AppendData(data []interface{}) *XlsxWrite {
x.rows = append(x.rows, &data)
return x
}
func (x *XlsxWrite) AppendRows(data [][]interface{}) *XlsxWrite {
for _, i := range data {
x.rows = append(x.rows, &i)
}
return x
}
func (x *XlsxWrite) SetSheetName(sheetName string) *XlsxWrite {
x.sheetName = sheetName
index, err := x.fh.NewSheet(x.sheetName)
if err != nil {
return nil
}
x.fh.SetActiveSheet(index)
return x
}
func (x *XlsxWrite) Save2File(filename string) (err error) {
err = x.fh.SetSheetRow(x.sheetName, "A1", x.titles)
if err != nil {
return err
}
for i := 0; i < len(x.rows); i++ {
err = x.fh.SetSheetRow(x.sheetName, fmt.Sprintf("A%d", i+2), x.rows[i])
if err != nil {
return err
}
}
if err = x.fh.SaveAs(filename); err != nil {
return err
}
return nil
}
func (x *XlsxWrite) OutputForGin(ctx *gin.Context, filename string) (err error) {
ctx.Header("Content-Type", "application/octet-stream")
ctx.Header("Content-Disposition", "attachment; filename="+filename)
ctx.Header("Content-Transfer-Encoding", "binary")
ctx.Header("Expires", "0")
return x.Output(ctx.Writer)
}
func (x *XlsxWrite) OutputResponseWriter(w http.ResponseWriter, filename string) (err error) {
header := w.Header()
header.Set("Content-Type", "application/octet-stream")
header.Set("Content-Disposition", "attachment; filename="+filename)
header.Set("Content-Transfer-Encoding", "binary")
header.Set("Expires", "0")
return x.Output(w)
}
func (x *XlsxWrite) Output(w io.Writer) (err error) {
//if x.Encoding == UTF8 {
// _, err = w.Write(BOM_UTF8)
// if err != nil {
// return err
// }
//} else if x.Encoding == UTF16 {
// _, err = w.Write(BOM_UTF16)
// if err != nil {
// return err
// }
//}
err = x.fh.SetSheetRow(x.sheetName, "A1", x.titles)
if err != nil {
return err
}
for i := 0; i < len(x.rows); i++ {
err = x.fh.SetSheetRow(x.sheetName, fmt.Sprintf("A%d", i+2), x.rows[i])
if err != nil {
return err
}
}
if err = x.fh.Write(w); err != nil {
return err
}
return nil
}
go-zero/etcd-configurator/config_center.go (1.1 KiB)
package etcd_configurator
import (
goetcd "github.com/gif-gif/go.io/go-etcd"
golog "github.com/gif-gif/go.io/go-log"
configurator "github.com/zeromicro/go-zero/core/configcenter"
"github.com/zeromicro/go-zero/core/configcenter/subscriber"
)
func NewConfigCenter[T any](key string, etcd goetcd.Config, callback func(t T)) (configurator.Configurator[T], error) {
ss := subscriber.MustNewEtcdSubscriber(subscriber.EtcdConf{
Hosts: etcd.Endpoints, // etcd 地址
User: etcd.Username,
Pass: etcd.Password,
Key: key, // 配置key
})
// 创建 configurator
cc := configurator.MustNewConfigCenter[T](configurator.Config{
Type: "json", // 配置值类型:json,yaml,toml
}, ss)
// 获取配置
// 注意: 配置如果发生变更,调用的结果永远获取到最新的配置
v, err := cc.GetConfig()
if err != nil {
return nil, err
}
callback(v)
// 如果想监听配置变化,可以添加 listener
cc.AddListener(func() {
v, err := cc.GetConfig()
if err != nil {
golog.WithTag("etcd").Fatal(err)
return
}
callback(v)
})
return cc, nil
}
go-zero/etcd-configurator/test/config_test.go (980 B)
package test
import (
"encoding/json"
"fmt"
gocontext "github.com/gif-gif/go.io/go-context"
goetcd "github.com/gif-gif/go.io/go-etcd"
etcd_configurator "github.com/gif-gif/go.io/go-zero/etcd-configurator"
"log"
"testing"
)
// 配置结构定义
type TestSt struct {
Name string `json:"name"`
}
func TestEtcdConfigListener(t *testing.T) {
etcd_configurator.NewConfigCenter[TestSt]("config-test", goetcd.Config{
Endpoints: []string{"127.0.0.1:2379"},
}, func(t TestSt) {
println(t.Name)
})
<-gocontext.WithCancel().Done()
}
func TestEtcdSaveConfig(t *testing.T) {
goetcd.Init(goetcd.Config{
Endpoints: []string{"127.0.0.1:2379"},
//Username: "root",
//Password: "123456",
})
goetcd.Del("config-test")
data := &TestSt{
Name: "Test112",
}
str, _ := json.Marshal(data)
if _, err := goetcd.Set("config-test", string(str)); err != nil {
log.Fatalln(err)
}
fmt.Println("Setting config:", goetcd.GetString("config-test"))
//Del("config-test")
}
go-zero/grpc-limit.go (905 B)
package gozero
import (
"context"
"github.com/zeromicro/go-zero/core/logx"
"github.com/zeromicro/go-zero/core/syncx"
"github.com/zeromicro/go-zero/zrpc"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"net/http"
)
// Gozero Grpc 调用限流 limit 是最大并发数
func GrpcLimit(s *zrpc.RpcServer, limit int) {
l := syncx.NewLimit(limit)
s.AddUnaryInterceptors(func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
if l.TryBorrow() {
defer func() {
if err := l.Return(); err != nil {
logx.Error(err)
}
}()
return handler(ctx, req)
} else {
logx.Errorf("concurrent connections over %d, rejected with code %d",
limit, http.StatusServiceUnavailable)
return nil, status.Error(codes.Unavailable, "concurrent connections over limit")
}
})
}
go-zero/loggerInterceptor.go (825 B)
package gozero
import (
"context"
goerror "github.com/gif-gif/go.io/go-error"
"github.com/pkg/errors"
"github.com/zeromicro/go-zero/core/logx"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func LoggerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
resp, err = handler(ctx, req)
if err != nil {
causeErr := errors.Cause(err) // err类型
if e, ok := causeErr.(*goerror.CodeError); ok { //自定义错误类型
logx.WithContext(ctx).Errorf("【RPC-SRV-ERR】 %+v", err)
//转成grpc err
err = status.Error(codes.Code(e.GetErrCode()), e.GetErrMsg())
} else {
logx.WithContext(ctx).Errorf("【RPC-SRV-ERR】 %+v", err)
}
}
return resp, err
}
go-zero/transactionex/ReadMe.md (665 B)
事务性操作 主要在go-zero 框架下执行
初始化 : TransactionModel: transactionex.NewModel(conn, cacheConf),
userInserter := func(session sqlx.Session) error {
return l.svcCtx.UserModel.InsertWithTx(l.ctx, session, newUserInfo)
}
deviceInserter := func(session sqlx.Session) error {
return l.svcCtx.DeviceModel.InsertWithTx(l.ctx, session, deviceInfo)
}
inserters := []transactionex.TableTransactionFunc{
userInserter,
deviceInserter,
}
err := l.svcCtx.TransactionModel.Transactions(inserters)
if err != nil {
return nil, errors.Wrapf(xerr.NewErrCode(xerr.DB_ERROR), "Register InsertWithTx error:%v in:%+v", err, in)
}
go-zero/transactionex/model.go (731 B)
package transactionx
import (
"github.com/zeromicro/go-zero/core/stores/cache"
"github.com/zeromicro/go-zero/core/stores/sqlc"
"github.com/zeromicro/go-zero/core/stores/sqlx"
)
type Model struct {
sqlc.CachedConn
}
type TableTransactionFunc func(session sqlx.Session) error
func NewModel(conn sqlx.SqlConn, c cache.CacheConf) *Model {
return &Model{
CachedConn: sqlc.NewConn(conn, c),
}
}
// Transaction 事务方式插入多个表的数据
func (m *Model) Transactions(inserters []TableTransactionFunc) error {
err := m.CachedConn.Transact(func(session sqlx.Session) error {
for _, inserter := range inserters {
err := inserter(session)
if err != nil {
return err
}
}
return nil
})
return err
}
go-zero/transactionex/transaction.go (916 B)
package transactionx
import (
"context"
"fmt"
"github.com/zeromicro/go-zero/core/stores/sqlx"
)
type TransactionHandler interface {
InsertTransactions(ctx context.Context, session sqlx.Session, info string) error
DeleteTransactions(ctx context.Context, session sqlx.Session, info string) error
UpdateTransactions(ctx context.Context, session sqlx.Session, info string) error
}
var (
transactionManager = TransactionManager{
TransactionHandlers: map[string]TransactionHandler{},
}
)
type TransactionManager struct {
TransactionHandlers map[string]TransactionHandler
}
func RegisterTransactionHandler(key string, handleFunc TransactionHandler) {
transactionManager.TransactionHandlers[key] = handleFunc
}
func GetTransactionHandler(key string) TransactionHandler {
key = fmt.Sprintf("`%s`", key)
if handler, ok := transactionManager.TransactionHandlers[key]; ok {
return handler
}
return nil
}
go-zero/transactionex/types.go (110 B)
package transactionx
const (
OperateInsert = "insert"
OperateUpdate = "update"
OperateDelete = "delete"
)
go.mod (10.1 KiB)
module github.com/gif-gif/go.io
go 1.24.1
require (
github.com/ClickHouse/clickhouse-go/v2 v2.30.0
github.com/IBM/sarama v1.43.3
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/andybalholm/brotli v1.1.1
github.com/apache/rocketmq-client-go/v2 v2.1.2
github.com/eclipse/paho.mqtt.golang v1.5.0
github.com/elastic/go-elasticsearch/v8 v8.14.0
github.com/fvbock/endless v0.0.0-20170109170031-447134032cb6
github.com/gin-contrib/pprof v1.5.0
github.com/gin-gonic/gin v1.10.0
github.com/go-co-op/gocron/v2 v2.16.3
github.com/go-playground/validator/v10 v10.20.0
github.com/go-resty/resty/v2 v2.15.3
github.com/gogf/gf v1.16.9
github.com/golang-jwt/jwt/v5 v5.0.0
github.com/google/go-querystring v1.1.0
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/hibiken/asynq v0.25.1
github.com/ip2location/ip2location-go/v9 v9.7.0
github.com/json-iterator/go v1.1.12
github.com/minio/minio-go/v7 v7.0.70
github.com/mojocn/base64Captcha v1.3.6
github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852
github.com/olivere/elastic/v7 v7.0.32
github.com/oschwald/geoip2-golang v1.11.0
github.com/panjf2000/ants/v2 v2.11.3
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.22.0
github.com/prometheus/common v0.62.0
github.com/redis/go-redis/v9 v9.7.0
github.com/samber/lo v1.47.0
github.com/shopspring/decimal v1.4.0
github.com/silenceper/wechat/v2 v2.1.6
github.com/stretchr/testify v1.10.0
github.com/xuri/excelize/v2 v2.9.1
github.com/zeromicro/go-zero v1.7.3
go.etcd.io/etcd/client/pkg/v3 v3.5.16
go.etcd.io/etcd/client/v3 v3.5.16
go.mongodb.org/mongo-driver v1.17.1
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.40.0
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
golang.org/x/oauth2 v0.24.0
golang.org/x/sync v0.16.0
golang.org/x/text v0.27.0
google.golang.org/api v0.202.0
google.golang.org/grpc v1.67.1
gopkg.in/telebot.v3 v3.3.8
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/clickhouse v0.6.1
gorm.io/driver/mysql v1.5.7
gorm.io/driver/postgres v1.5.9
gorm.io/driver/sqlite v1.5.6
gorm.io/driver/sqlserver v1.5.3
gorm.io/gorm v1.25.10
)
require (
cloud.google.com/go/auth v0.9.8 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go/compute/metadata v0.5.2 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/ClickHouse/ch-go v0.61.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/eapache/go-resiliency v1.7.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/fatih/color v1.17.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.6.0 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.7.6 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jonboulle/clockwork v0.5.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/microsoft/go-mssqldb v1.6.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/onsi/ginkgo/v2 v2.17.2 // indirect
github.com/onsi/gomega v1.33.1 // indirect
github.com/openzipkin/zipkin-go v0.4.3 // indirect
github.com/oschwald/maxminddb-golang v1.13.1 // indirect
github.com/paulmach/orb v0.11.1 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.4 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/cast v1.7.0 // indirect
github.com/tidwall/gjson v1.17.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tiendc/go-deepcopy v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xuri/efp v0.0.1 // indirect
github.com/xuri/nfp v0.0.1 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.etcd.io/etcd/api/v3 v3.5.16 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.29.0 // indirect
go.opentelemetry.io/otel/sdk v1.27.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/image v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/term v0.33.0 // indirect
golang.org/x/time v0.8.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
google.golang.org/protobuf v1.36.5 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/api v0.29.3 // indirect
k8s.io/apimachinery v0.29.4 // indirect
k8s.io/client-go v0.29.3 // indirect
k8s.io/klog/v2 v2.110.1 // indirect
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
lukechampine.com/uint128 v1.3.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
goio/README.md (668 B)
go.io
framework interfaces and configurations for libraries
package main
import (
"github.com/gif-gif/go.io/goio"
"github.com/gif-gif/go.io/goio/test/router"
)
func main() {
startSever()
}
func startSever() {
goio.Init(goio.DEVELOPMENT)
s := goio.NewServer(
goio.ServerNameOption("serverName"),
goio.EnvOption(goio.Env),
//goio.EnableEncryptionOption(),
goio.PProfEnableOption(false),
goio.NoLogPathsOption("/captcha/get"),
)
// 路由
router.Routes(s.Group("/"))
// 启动
s.Run(":1000")
}
curl --location --request POST 'http://127.0.0.1:1000/health' \
--header 'User-Agent: Apifox/1.0.0 (https://apifox.com)'
goio/env.go (1.7 KiB)
package goio
import (
"fmt"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/go-log/adapters"
gomessage "github.com/gif-gif/go.io/go-message"
)
type Environment string
// 库运行模式 dev|test|rt|pre|pro go-zero
const (
PRODUCTION Environment = "pro"
PRE Environment = "pre"
RT Environment = "rt"
TEST Environment = "test"
DEVELOPMENT Environment = "dev"
)
var (
envTags = map[Environment]string{
PRODUCTION: "prod",
PRE: "pre",
RT: "rt",
TEST: "test",
DEVELOPMENT: "dev",
}
)
func (env Environment) String() string {
return string(env)
}
func (env Environment) Tag() string {
return envTags[env]
}
// 日志级别一键设置, 测试环境和生产环境发送飞书消息
//
// 也可以不用这个方法,项目中可以自定义, golog.WithHook 多次调用不会覆盖,每次调用都会生效
func Setup(feishu string) {
// 发送飞书消息
if feishu == "" {
return
}
golog.WithHook(func(msg *golog.Message) {
if msg.Level <= golog.WARN {
return
}
if IsTest() || IsPro() || IsPre() {
gomessage.FeiShu(feishu, fmt.Sprintf(">> %s/%s >> %s", Name, Env.String(), string(msg.JSON())))
}
})
}
func SetupLogDefault() {
if IsTest() || IsPro() || IsPre() {
golog.SetAdapter(adapters.NewFileAdapter())
}
}
func SetupLogDefaultByFilepath(filepath string) {
if IsTest() || IsPro() || IsPre() {
golog.SetAdapter(adapters.NewFileAdapter(adapters.FilePathOption(filepath)))
}
}
func IsTest() bool {
return Env == TEST
}
func IsPro() bool {
return Env == PRODUCTION
}
func IsDev() bool {
return Env == DEVELOPMENT
}
func IsPre() bool {
return Env == PRE
}
goio/error_state.go (374 B)
package goio
const SUCCESS_REQUEST = 0
// 全局错误码
// client
const ERROR_REQUEST int64 = 400
const ERROR_UNAUTHORIZED int64 = 401
const ERROR_FORBIDDEN int64 = 403
const ERROR_NOTFOUND int64 = 404
const ERROR_INVALID int64 = -1
// server
const ERROR_SERVER int64 = 500
const ERROR_DATABASE int64 = 555
const ERROR_REDIS int64 = 556
const ERROR_PARAMS int64 = 700
goio/flag.go (616 B)
package goio
import (
"flag"
"fmt"
"os"
)
/**
- deploy.sh
// 定义
version=v1.0.1
buildVersion="${version}.$(date +%Y%m%d).$(date +%H%M)"
// 编译
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X https://github.com/gif-gif/go.io/goio.Version=$buildVersion" -o ss
- 执行
./ss -v
*/
var (
Version string
VersionFlag = flag.Bool("v", false, "version")
HelpFlag = flag.Bool("h", false, "help")
)
func FlagInit() {
if !flag.Parsed() {
flag.Parse()
}
if *VersionFlag {
fmt.Println(Version)
os.Exit(0)
}
if *HelpFlag {
flag.PrintDefaults()
os.Exit(0)
}
}
goio/go-test/main.go (543 B)
package main
import "fmt"
func main() {
testT()
}
// 泛型类型别名预览示例
// 使用泛型类型别名
func testT() {
// 定义一个泛型类型别名,表示任意类型T的指针
type Ptr[T any] *T
// 定义一个泛型类型别名,表示任意类型T的切片
type Slice[T any] []T
// 创建一个int类型的指针
var p Ptr[int] = new(int)
*p = 42
// 创建一个string类型的切片
var s Slice[string] = []string{"Hello", "World"}
fmt.Println(*p) // 输出: 42
fmt.Println(s) // 输出: [Hello World]
}
goio/goi/clickhouse.go (514 B)
package goi
import (
"database/sql"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
goclickhouse "github.com/gif-gif/go.io/go-db/go-clickhouse"
)
func CkClient(names ...string) *goclickhouse.GoClickHouse {
return goclickhouse.GetClient(names...)
}
func CK(names ...string) driver.Conn {
client := CkClient(names...)
if client == nil {
return nil
}
return client.Conn()
}
func CKDB(names ...string) *sql.DB {
client := CkClient(names...)
if client == nil {
return nil
}
return client.DB()
}
goio/goi/db.go (290 B)
package goi
import (
"github.com/gif-gif/go.io/go-db/gogorm"
"gorm.io/gorm"
)
func DbClient(names ...string) *gogorm.GoGorm {
return gogorm.GetClient(names...)
}
func DB(names ...string) *gorm.DB {
client := DbClient(names...)
if client == nil {
return nil
}
return client.DB
}
goio/goi/es.go (282 B)
package goi
import (
goes "github.com/gif-gif/go.io/go-db/go-es"
"github.com/olivere/elastic/v7"
)
func EsClient(names ...string) *goes.GoEs {
return goes.GetClient(names...)
}
func ES(names ...string) *elastic.Client {
client := EsClient(names...)
return client.Client()
}
goio/goi/etcd.go (155 B)
package goi
import (
goetcd "github.com/gif-gif/go.io/go-etcd"
)
func Etcd(names ...string) *goetcd.GoEtcdClient {
return goetcd.GetClient(names...)
}
goio/goi/goip.go (139 B)
package goi
import (
goip "github.com/gif-gif/go.io/go-ip"
)
func GoIp(names ...string) *goip.GoIp {
return goip.GetClient(names...)
}
goio/goi/gominio.go (165 B)
package goi
import (
gominio "github.com/gif-gif/go.io/go-oss/go-minio"
)
func GoMinio(names ...string) *gominio.Uploader {
return gominio.GetClient(names...)
}
goio/goi/jwt.go (152 B)
package goi
import (
gojwt "github.com/gif-gif/go.io/go-sso/go-jwt"
)
func GoJwt(names ...string) *gojwt.GoJwt {
return gojwt.GetClient(names...)
}
goio/goi/kafka.go (363 B)
package goi
import (
gokafka "github.com/gif-gif/go.io/go-mq/go-kafka"
)
func Kafka(names ...string) *gokafka.GoKafka {
return gokafka.GetClient(names...)
}
func Producer(names ...string) gokafka.IProducer {
return gokafka.GetClient(names...).Producer()
}
func Consumer(names ...string) gokafka.IConsumer {
return gokafka.GetClient(names...).Consumer()
}
goio/goi/mongo.go (342 B)
package goi
import (
gomongo "github.com/gif-gif/go.io/go-db/go-mongo"
"go.mongodb.org/mongo-driver/mongo"
)
func Mongo(names ...string) *mongo.Database {
client := MongoClient(names...)
if client == nil {
return nil
}
return client.DB()
}
func MongoClient(names ...string) *gomongo.GoMongo {
return gomongo.GetClient(names...)
}
goio/goi/readme.md (18 B)
goio Interfaces
goio/goi/redis.go (333 B)
package goi
import (
goredis "github.com/gif-gif/go.io/go-db/go-redis"
goredisc "github.com/gif-gif/go.io/go-db/go-redis/go-redisc"
)
// 单机
func Redis(names ...string) *goredis.GoRedis {
return goredis.GetClient(names...)
}
// 集群
func Redisc(names ...string) *goredisc.GoRedisC {
return goredisc.GetClient(names...)
}
goio/goio.go (474 B)
package goio
// 当前运行环境
var Env Environment = "dev" //string `json:",default=pro,options=dev|test|rt|pre|pro"`
var Name string
func Init(env Environment, name ...string) {
Env = env
if l := len(name); l > 0 {
for i, n := range name {
if n != "" {
if i == 0 {
Name = n
} else {
Name += "_" + n
}
break
}
}
} else {
Name = "default"
}
//golog.SetAdapter(golog.NewFileAdapter()) //默认当前工程目录logs/date.log
}
goio/server/config.go (1.9 KiB)
package goserver
import (
"encoding/json"
goclickhouse "github.com/gif-gif/go.io/go-db/go-clickhouse"
goes "github.com/gif-gif/go.io/go-db/go-es"
gomongo "github.com/gif-gif/go.io/go-db/go-mongo"
goredis "github.com/gif-gif/go.io/go-db/go-redis"
"github.com/gif-gif/go.io/go-db/gogorm"
goetcd "github.com/gif-gif/go.io/go-etcd"
golog "github.com/gif-gif/go.io/go-log"
gokafka "github.com/gif-gif/go.io/go-mq/go-kafka"
"github.com/gif-gif/go.io/go-utils/goprometheus/goprometheusx"
"github.com/gif-gif/go.io/goio"
"gopkg.in/yaml.v3"
"os"
)
type Config struct {
Env goio.Environment `yaml:"env"`
Server struct {
Addr string `yaml:"addr"`
Name string `yaml:"name"`
} `yaml:"server"`
Prometheus goprometheusx.Config `yaml:"prometheus"`
MongoDB gomongo.Config `yaml:"mongodb"`
Mysql gogorm.Config `yaml:"mysql"`
Postgres gogorm.Config `yaml:"postgres"`
Sqlite gogorm.Config `yaml:"sqlite"`
Clickhouse1 gogorm.Config `yaml:"clickhouse1"`
Redis goredis.Config `yaml:"redis"`
Kafka gokafka.Config `yaml:"kafka"`
Clickhouse goclickhouse.Config `yaml:"clickhouse"`
Es goes.Config `yaml:"es"`
////EsIndex EsIndex `yaml:"es_index"`
Etcd goetcd.Config `yaml:"etcd"`
FeiShu string `yaml:"feishu"`
}
func LoadYamlConfig(yamlFile string, conf interface{}) (err error) {
if yamlFile == "" {
yamlFile = "api-local.yaml"
}
var buf []byte
buf, err = os.ReadFile(yamlFile)
if err != nil {
golog.Error(err.Error())
return
}
if err = yaml.Unmarshal(buf, conf); err != nil {
golog.Error(err.Error())
}
return
}
func LoadJsonConfig(jsonFile string, conf interface{}) (err error) {
if jsonFile == "" {
jsonFile = "api-local.json"
}
var buf []byte
buf, err = os.ReadFile(jsonFile)
if err != nil {
golog.Error(err.Error())
return
}
if err = json.Unmarshal(buf, conf); err != nil {
golog.Error(err.Error())
}
return
}
goio/server/server.go (4.3 KiB)
package goserver
import (
"bytes"
"fmt"
"github.com/fvbock/endless"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gin-contrib/pprof"
"github.com/gin-gonic/gin"
"io"
"io/ioutil"
"os"
"strings"
"time"
)
// 自己定义web服务
type Server struct {
*gin.Engine
}
func NewServer(opt ...Option) *Server {
for _, o := range opt {
o.apply(defaultOptions)
}
s := &Server{
Engine: gin.New(),
}
s.Engine.NoRoute(s.noRoute)
s.Engine.NoMethod(s.noMethod)
s.Use(s.cors, s.noAccess, s.setFields, s.encrypt, s.log, s.recovery)
return s
}
// 启动服务
func (s *Server) Run(addr string) {
pid := fmt.Sprintf("%d", os.Getpid())
if err := ioutil.WriteFile(".pid", []byte(pid), 0644); err != nil {
golog.Panic(err.Error())
}
// 性能分析
if defaultOptions.pprofEnable {
pprof.Register(s.Engine, "/goio/pprof")
}
endless.NewServer(addr, s.Engine).ListenAndServe()
}
// 跨域
func (s *Server) cors(c *gin.Context) {
c.Header("Access-Control-Allow-Origin", "*")
c.Header("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
c.Header("Access-Control-Allow-Headers", strings.Join(defaultOptions.corsHeaders, ","))
c.Next()
}
// 禁止访问
func (s *Server) noAccess(c *gin.Context) {
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(200)
return
}
if _, ok := defaultOptions.noAccessPath[c.Request.URL.Path]; ok {
c.AbortWithStatus(200)
return
}
c.Next()
}
// 设置字段
func (s *Server) setFields(c *gin.Context) {
c.Set("__begin_time", time.Now())
c.Set("__server_name", defaultOptions.serverName)
c.Set("__env", defaultOptions.env.Tag())
c.Next()
}
// 加解密
func (s *Server) encrypt(c *gin.Context) {
if !defaultOptions.encryptionEnable {
c.Next()
return
}
switch strings.ToUpper(c.Request.Method) {
case "POST", "PUT":
default:
c.Next()
return
}
switch strings.ToLower(c.Request.Header.Get("Content-Type")) {
case "multipart/form-data":
c.Next()
return
}
if _, ok := defaultOptions.encryptionExcludeUris[c.Request.RequestURI]; ok {
c.Next()
return
}
var buf bytes.Buffer
io.Copy(&buf, c.Request.Body)
b, err := defaultOptions.encryption.Decode(buf.String())
if err != nil {
s.abortWithStatus50X(c, 5002, "解码失败,原因:"+err.Error())
return
}
if l := len(b); l == 0 {
c.Next()
return
}
c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(b))
c.Next()
}
// log
func (s *Server) log(c *gin.Context) {
if _, ok := defaultOptions.noLogPath[c.Request.RequestURI]; ok {
c.Next()
return
}
beginTime := c.GetTime("__begin_time")
header := gin.H{}
if v := c.GetHeader("Authorization"); v != "" {
header["authorization"] = v
}
if v := c.GetHeader("Content-Type"); v != "" {
header["content-type"] = v
}
req := gin.H{
"method": c.Request.Method,
"uri": c.Request.RequestURI,
"header": header,
"client-ip": ClientIP(c),
"gotrace-id": RequestId(c),
}
if v := RequestBody(c); v != nil {
req["body"] = v
}
l := golog.WithTag("goio-api").
WithField("request", req)
c.Next()
if !beginTime.IsZero() {
l.WithField("duration", fmt.Sprintf("%dms", time.Since(beginTime)/1e6))
}
ctx := c.Copy()
for k, v := range ctx.Keys {
if k == "__response" {
continue
}
l.WithField(k, v)
}
if resp, has := ctx.Get("__response"); has {
if r, ok := resp.(*Response); resp != nil && ok {
l.WithField("response", resp)
if r == nil {
l.Error(resp)
return
}
if r.Success {
return
}
l.Error(r.ErrorCode, r.ErrorMessage, r.ShowType, r.TraceId, r.Host)
}
}
l.Debug()
}
// 捕获panic信息
func (s *Server) recovery(c *gin.Context) {
defer func() {
if r := recover(); r != nil {
s.abortWithStatus50X(c, 5001, fmt.Sprintf("请求异常, 提示信息: %v", r))
}
}()
c.Next()
}
// 找不到路由
func (s *Server) noRoute(c *gin.Context) {
s.abortWithStatus40X(c, 404, "Page Not Found")
}
// 找不到方法
func (s *Server) noMethod(c *gin.Context) {
s.abortWithStatus40X(c, 405, "Method not allowed")
}
func (*Server) abortWithStatus40X(c *gin.Context, code int32, msg string) {
resp := Error(code, msg, msg)
c.Set("__response", resp)
c.AbortWithStatusJSON(int(code), resp)
}
func (*Server) abortWithStatus50X(c *gin.Context, code int32, msg string) {
resp := Error(code, msg, msg)
c.Set("__response", resp)
c.AbortWithStatusJSON(500, resp)
}
goio/server/server_encryption.go (736 B)
package goserver
import (
"encoding/hex"
goutils "github.com/gif-gif/go.io/go-utils"
"strings"
)
type Encryption struct {
Key string
Secret string
}
func (enc *Encryption) Encode(b []byte) (str string, err error) {
if l := len(b); l == 0 {
return
}
var bts []byte
bts, err = goutils.AESCBCEncrypt(b, []byte(enc.Key), []byte(enc.Secret))
if err != nil {
return
}
str = hex.EncodeToString(bts)
return
}
func (enc *Encryption) Decode(str string) (b []byte, err error) {
str = strings.ReplaceAll(str, "\"", "")
if l := len(str); l == 0 {
return
}
var bts []byte
bts, err = hex.DecodeString(str)
if err != nil {
return
}
b, err = goutils.AESCBCDecrypt(bts, []byte(enc.Key), []byte(enc.Secret))
return
}
goio/server/server_encryption_test.go (657 B)
package goserver
import (
"log"
"testing"
)
func TestEncryption_Encode(t *testing.T) {
enc := &Encryption{
Key: "18586555d498e2c3d0e0baaf0bdde3ce",
Secret: "7732c2d83bad98173790cd62483cde791df5ce6e62247dc563a15367e4750340",
}
str, err := enc.Encode([]byte("123adbasdf"))
if err != nil {
log.Println(err)
return
}
log.Println(str)
}
func TestEncryption_Decode(t *testing.T) {
enc := &Encryption{
Key: "f455c40c0303189b7a1fe769f7689a52",
Secret: "aa4ef454a2c8d157fa5ea43df5357806",
}
b, err := enc.Decode("269e4be01b92d9f4c0a6477a4e5a3b49")
if err != nil {
log.Println(err.Error())
return
}
log.Println(string(b))
}
goio/server/server_grpc.go (1.4 KiB)
package goserver
import (
"context"
"fmt"
"github.com/gin-gonic/gin"
"runtime"
"strings"
"google.golang.org/grpc/metadata"
)
func prettyFile(file string) string {
index := strings.LastIndex(file, "/")
if index < 0 {
return file
}
index2 := strings.LastIndex(file[:index], "/")
if index2 < 0 {
return file[index+1:]
}
return file[index2+1:]
}
// 追踪信息
func trace(skip int) (arr []string) {
arr = []string{}
if skip == 0 {
skip = 1
}
for i := skip; i < 16; i++ {
_, file, line, _ := runtime.Caller(i)
if file == "" {
continue
}
if strings.Contains(file, ".pb.go") ||
strings.Contains(file, "runtime/") ||
strings.Contains(file, "src/testing") ||
strings.Contains(file, "pkg/mod/") ||
strings.Contains(file, "vendor/") {
continue
}
arr = append(arr, fmt.Sprintf("%s %dL", prettyFile(file), line))
}
return
}
func GrpcContext(c *gin.Context) context.Context {
md := metadata.New(map[string]string{})
if c != nil {
if v := c.GetString("__server_name"); v != "" {
md.Set("server-name", v)
}
if v := c.GetString("__env"); v != "" {
md.Set("env", v)
}
if v := RequestId(c); v != "" {
md.Set("gotrace-id", v)
}
if v := RequestId(c); v != "" {
arr := trace(2)
if l := len(arr); l > 0 {
md.Set("caller", strings.Join(arr, ", "))
}
}
}
return metadata.NewOutgoingContext(context.TODO(), md)
}
goio/server/server_handler.go (1.4 KiB)
package goserver
import (
"encoding/json"
"fmt"
"github.com/gin-gonic/gin"
"strings"
"time"
)
// 定义控制器抽象类
type iController interface {
DoHandle(c *gin.Context) *Response
}
// 定义控制器调用实现
func Handler(controller iController) gin.HandlerFunc {
return func(c *gin.Context) {
resp := controller.DoHandle(c)
if resp == nil {
return
}
c.Set("__response", resp.Copy())
// 计算执行时间
beginTime := c.GetTime("__begin_time")
if !beginTime.IsZero() {
c.Header("X-Response-Duration", fmt.Sprintf("%dms", time.Since(beginTime)/1e6))
}
if !defaultOptions.encryptionEnable {
c.JSON(200, resp)
return
}
switch strings.ToUpper(c.Request.Method) {
case "POST", "PUT":
default:
c.JSON(200, resp)
return
}
switch strings.ToLower(c.Request.Header.Get("Content-Type")) {
case "multipart/form-data":
c.JSON(200, resp)
return
}
if _, ok := defaultOptions.encryptionExcludeUris[c.Request.RequestURI]; ok {
c.JSON(200, resp)
return
}
b, err := json.Marshal(&resp.Data)
if err != nil {
c.JSON(500, Error(5003, "数据解析失败,原因:"+err.Error()))
return
}
body, err := defaultOptions.encryption.Encode(b)
if err != nil {
c.JSON(500, Error(5004, "数据解析失败,原因:"+err.Error()))
return
}
resp.Data = body
c.JSON(200, resp)
return
}
}
goio/server/server_options.go (2.4 KiB)
package goserver
import "github.com/gif-gif/go.io/goio"
var defaultOptions = &options{
noAccessPath: map[string]struct{}{
"/favicon.ico": {},
},
noLogPath: map[string]struct{}{
"/favicon.ico": {},
},
corsHeaders: []string{
"Content-Type", "Content-Length",
"Accept", "Referer", "User-Agent", "Authorization",
"X-Requested-Id", "X-Request-Timestamp", "X-Request-Sign",
"X-Request-AppId", "X-Request-Source", "X-Request-Token",
"X-Client-Id", "X-Client-Token",
},
encryptionExcludeUris: map[string]struct{}{},
}
type options struct {
pprofEnable bool
serverName string
env goio.Environment
corsHeaders []string
noAccessPath map[string]struct{}
noLogPath map[string]struct{}
encryption *Encryption
encryptionEnable bool
encryptionExcludeUris map[string]struct{}
}
type Option interface {
apply(opts *options)
}
type funcOption struct {
f func(opts *options)
}
func newFuncOption(f func(opts *options)) *funcOption {
return &funcOption{f: f}
}
func (f funcOption) apply(opts *options) {
f.f(opts)
}
// 开启分析
func PProfEnableOption(pprofEnable bool) Option {
return newFuncOption(func(opts *options) {
opts.pprofEnable = pprofEnable
})
}
// 服务名称
func ServerNameOption(serverName string) Option {
return newFuncOption(func(opts *options) {
opts.serverName = serverName
})
}
// 运行环境
func EnvOption(env goio.Environment) Option {
return newFuncOption(func(opts *options) {
opts.env = env
})
}
// 跨域
func CorsHeaderOption(corsHeaders ...string) Option {
return newFuncOption(func(opts *options) {
opts.corsHeaders = append(opts.corsHeaders, corsHeaders...)
})
}
// 禁止访问的path
func NoAccessPathsOption(noAccessPaths ...string) Option {
return newFuncOption(func(opts *options) {
for _, i := range noAccessPaths {
opts.noAccessPath[i] = struct{}{}
}
})
}
// 不记录日志的path
func NoLogPathsOption(noLogPaths ...string) Option {
return newFuncOption(func(opts *options) {
for _, i := range noLogPaths {
opts.noLogPath[i] = struct{}{}
}
})
}
// 启用加密传输
func EnableEncryptionOption(encryptKey, encryptSecret string, excludeUris ...string) Option {
return newFuncOption(func(opts *options) {
opts.encryptionEnable = true
opts.encryption = &Encryption{Key: encryptKey, Secret: encryptSecret}
for _, uri := range excludeUris {
opts.encryptionExcludeUris[uri] = struct{}{}
}
})
}
goio/server/server_response.go (4.0 KiB)
package goserver
import (
"encoding/json"
"fmt"
goerror "github.com/gif-gif/go.io/go-error"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/go-playground/validator/v10"
"github.com/gogf/gf/util/gconv"
"github.com/samber/lo"
"strings"
)
type Response struct {
Success bool `json:"success"`
Data any `json:"data"`
ErrorCode string `json:"errorCode"`
ErrorMessage string `json:"errorMessage"`
ShowType uint32 `json:"showType"`
TraceId string `json:"traceId"`
Host string `json:"host"`
}
// ResponseBuilder 是 Response 的构建器
type ResponseBuilder struct {
response *Response
}
// NewResponseBuilder 创建一个新的 Response 构建器
func NewResponseBuilder() *ResponseBuilder {
return &ResponseBuilder{
response: &Response{
Success: true, // 默认设置为成功
},
}
}
// WithSuccess 设置成功状态
func (b *ResponseBuilder) WithSuccess(success bool) *ResponseBuilder {
b.response.Success = success
return b
}
// WithData 设置数据
func (b *ResponseBuilder) WithData(data any) *ResponseBuilder {
b.response.Data = data
return b
}
// WithErrorCode 设置错误代码
func (b *ResponseBuilder) WithErrorCode(errorCode string) *ResponseBuilder {
b.response.ErrorCode = errorCode
return b
}
// WithErrorMessage 设置错误信息
func (b *ResponseBuilder) WithErrorMessage(errorMessage string) *ResponseBuilder {
b.response.ErrorMessage = errorMessage
return b
}
// WithShowType 设置显示类型
func (b *ResponseBuilder) WithShowType(showType uint32) *ResponseBuilder {
b.response.ShowType = showType
return b
}
// WithTraceId 设置追踪ID
func (b *ResponseBuilder) WithTraceId(traceId string) *ResponseBuilder {
b.response.TraceId = traceId
return b
}
// WithHost 设置主机信息
func (b *ResponseBuilder) WithHost(host string) *ResponseBuilder {
b.response.Host = host
return b
}
// Build 构建并返回最终的 Response 对象
func (b *ResponseBuilder) Build() *Response {
return b.response
}
// 便捷方法: 快速创建成功响应
func SuccessResponse(data any) *Response {
return NewResponseBuilder().
WithSuccess(true).
WithData(data).
Build()
}
// 便捷方法: 快速创建错误响应
func ErrorResponseX(errorCode string, errorMessage string, showType ...uint32) *Response {
return NewResponseBuilder().
WithSuccess(false).
WithErrorCode(errorCode).
WithErrorMessage(errorMessage).
WithShowType(lo.If[uint32](len(showType) == 0, 0).Else(showType[0])).
Build()
}
func ErrorResponse(err *goerror.CodeError) *Response {
return NewResponseBuilder().
WithSuccess(false).
WithErrorCode(gconv.String(err.GetErrCode())).
WithErrorMessage(err.GetErrMsg()).
WithShowType(err.GetShowType()).
WithTraceId(err.GetTraceId()).
WithHost(err.GetHost()).
Build()
}
func (rsp *Response) Copy() *Response {
r := NewResponseBuilder().
WithSuccess(rsp.Success).
WithData(rsp.Data).
WithErrorCode(rsp.ErrorCode).
WithErrorMessage(rsp.ErrorMessage).
WithShowType(rsp.ShowType).
WithTraceId(rsp.TraceId).
WithHost(rsp.Host).
Build()
return r
}
func (rsp *Response) String() string {
buf, err := json.Marshal(rsp)
if err != nil {
return err.Error()
}
return string(buf)
}
// Error 创建带格式化错误消息的错误响应
func Error(code int32, message string, v ...interface{}) *Response {
return NewResponseBuilder().
WithSuccess(false).
WithErrorCode(fmt.Sprintf("%d", code)).
WithErrorMessage(fmt.Sprintf(message, v...)).
Build()
}
func ErrorWithValidate(err error, messages map[string]string) *Response {
if v, ok := err.(*json.UnmarshalTypeError); ok {
return Error(7001, fmt.Sprintf("请求参数 %s 的类型是 %s, 不是 %s", v.Field, v.Type, v.Value))
}
if v, ok := err.(validator.ValidationErrors); ok {
for _, i := range v {
field := goutils.Camel2Case(i.Field())
key := fmt.Sprintf("%s_%s", field, strings.ToLower(i.Tag()))
if msg, ok := messages[key]; ok {
return Error(7002, msg)
}
return Error(7003, fmt.Sprintf("%s %s", field, i.Tag()))
}
}
return Error(7004, "参数错误", err)
}
goio/server/server_token.go (1.4 KiB)
package goserver
import (
"encoding/json"
"errors"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
"time"
)
type Token struct {
AppId string `json:"app_id"`
OpenId int64 `json:"data"`
NonceStr string `json:"nonce"`
Timestamp int64 `json:"ts"`
}
func (t *Token) Bytes() []byte {
buf, _ := json.Marshal(t)
return buf
}
func (t *Token) String() string {
return string(t.Bytes())
}
func CreateToken(appId string, openid int64) (tokenStr string, err error) {
token := &Token{
AppId: appId,
OpenId: openid,
NonceStr: goutils.NonceStr(),
Timestamp: time.Now().Unix(),
}
var (
key = goutils.MD5([]byte(appId))
iv = key[8:24]
encBuf []byte
)
encBuf, err = goutils.AESCBCEncrypt(token.Bytes(), []byte(key), []byte(iv))
if err != nil {
golog.Error(err.Error())
return
}
tokenStr = goutils.Base64Encode(encBuf)
return
}
func ParseToken(tokenStr, appId string) (token *Token, err error) {
var (
tokenBuf = goutils.Base64Decode(tokenStr)
key = goutils.MD5([]byte(appId))
iv = key[8:24]
b []byte
)
b, err = goutils.AESCBCDecrypt(tokenBuf, []byte(key), []byte(iv))
if err != nil {
golog.Error(err.Error())
return
}
token = new(Token)
if err = json.Unmarshal(b, token); err != nil {
golog.Error(err.Error())
return
}
if token.AppId != appId {
err = errors.New("appid invalid")
}
return
}
goio/server/server_upload.go (1.2 KiB)
package goserver
import (
"fmt"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gin-gonic/gin"
"io/ioutil"
"os"
"path"
)
type LocalUpload struct {
}
func (lu LocalUpload) Upload(c *gin.Context, uploadDir string) *Response {
f, fh, err := c.Request.FormFile("file")
if err != nil {
return Error(7001, fmt.Sprintf("上传失败,原因:%s", err.Error()))
}
data, err := ioutil.ReadAll(f)
if err != nil {
return Error(7002, fmt.Sprintf("上传失败,原因:%s", err.Error()))
}
f.Close()
md5str := goutils.MD5(data)
filepath := md5str[0:2] + "/" + md5str[2:4] + "/"
if err := os.MkdirAll(uploadDir+filepath, 0755); err != nil {
return Error(7003, fmt.Sprintf("上传失败,原因:%s", err.Error()))
}
fileExt := path.Ext(fh.Filename)
fileBasename := path.Base(fh.Filename)
filename := filepath + fileBasename + "_" + md5str[8:16] + fileExt
ff, err := os.Create(uploadDir + filename)
if err != nil {
return Error(7004, fmt.Sprintf("上传失败,原因:%s", err.Error()))
}
defer ff.Close()
if _, err := ff.Write(data); err != nil {
return Error(7005, fmt.Sprintf("上传失败,原因:%s", err.Error()))
}
return SuccessResponse(gin.H{
"url": filename,
})
}
goio/server/server_utils.go (2.4 KiB)
package goserver
import (
"bytes"
"encoding/json"
"github.com/gin-gonic/gin"
"github.com/gogf/gf/util/gconv"
"github.com/google/uuid"
"io"
"io/ioutil"
)
// 唯一ID
func RequestId(c *gin.Context) string {
if v := c.GetHeader("X-Request-Id"); v != "" {
return v
}
if v := c.Query("request_id"); v != "" {
return v
}
if v := c.GetHeader("X-Trace-Id"); v != "" {
return v
}
if v := c.Query("trace_id"); v != "" {
return v
}
return uuid.New().String()
}
// 客户端IP
func ClientIP(c *gin.Context) string {
if v := c.GetHeader("X-Real-IP"); v != "" {
return v
}
if v := c.GetHeader("X-Forwarded-For"); v != "" {
return v
}
if v := c.ClientIP(); v == "::1" {
return "127.0.0.1"
}
return ""
}
// 请求数据
func RequestBody(c *gin.Context) interface{} {
var (
b []byte
buf bytes.Buffer
contentType = c.ContentType()
)
switch contentType {
case "application/x-www-form-urlencoded", "text/xml", "application/json":
io.Copy(&buf, c.Request.Body)
b = buf.Bytes()
c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(b))
default:
return nil
}
if contentType == "application/json" {
var body interface{}
if err := json.Unmarshal(b, &body); err == nil {
return body
}
}
return string(b)
}
// 请求成功
func HttpSuccess(data interface{}) *Response {
return SuccessResponse(data)
}
// 请求格式错误,比如参数格式、参数字段名等 不正确
func HttpBadRequest(msg string, showType ...uint32) *Response {
return ErrorResponseX(gconv.String(400), msg, showType...)
}
// 用户没有访问权限,需要进行身份认证
func HttpUnauthorized(msg string, showType ...uint32) *Response {
return ErrorResponseX(gconv.String(401), msg, showType...)
}
// 用户已进行身份认证,但权限不够
func HttpForbidden(msg string, showType ...uint32) *Response {
return ErrorResponseX(gconv.String(403), msg, showType...)
}
// 接口不存在
func HttpNotFound(msg string, showType ...uint32) *Response {
return ErrorResponseX(gconv.String(404), msg, showType...)
}
// 服务器内部错误
func HttpServerError(msg string, showType ...uint32) *Response {
return ErrorResponseX(gconv.String(500), msg, showType...)
}
// 如需返回特殊错误码,调用此接口
func HttpFailForCode(code int64, msg string, showType ...uint32) *Response {
return ErrorResponseX(gconv.String(code), msg, showType...)
}
goio/server/server_validator.go (687 B)
package goserver
import (
"encoding/json"
"fmt"
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/go-playground/validator/v10"
"strings"
)
func ValidationMessage(err error, messages map[string]string) string {
if v, ok := err.(*json.UnmarshalTypeError); ok {
return fmt.Sprintf("请求参数 %s 的类型是 %s, 不是 %s", v.Field, v.Type, v.Value)
}
if v, ok := err.(validator.ValidationErrors); ok {
for _, i := range v {
field := goutils.Camel2Case(i.Field())
key := fmt.Sprintf("%s_%s", field, strings.ToLower(i.Tag()))
if msg, ok := messages[key]; ok {
return msg
}
return fmt.Sprintf("%s %s", field, i.Tag())
}
}
return err.Error()
}
goio/server-case/config/config.go (1.9 KiB)
package conf
import (
"encoding/json"
goclickhouse "github.com/gif-gif/go.io/go-db/go-clickhouse"
goes "github.com/gif-gif/go.io/go-db/go-es"
gomongo "github.com/gif-gif/go.io/go-db/go-mongo"
goredis "github.com/gif-gif/go.io/go-db/go-redis"
"github.com/gif-gif/go.io/go-db/gogorm"
goetcd "github.com/gif-gif/go.io/go-etcd"
golog "github.com/gif-gif/go.io/go-log"
gokafka "github.com/gif-gif/go.io/go-mq/go-kafka"
"github.com/gif-gif/go.io/go-utils/goprometheus/goprometheusx"
"github.com/gif-gif/go.io/goio"
"gopkg.in/yaml.v3"
"os"
)
type Config struct {
Env goio.Environment `yaml:"env"`
Server struct {
Addr string `yaml:"addr"`
Name string `yaml:"name"`
} `yaml:"server"`
Prometheus goprometheusx.Config `yaml:"prometheus"`
MongoDB gomongo.Config `yaml:"mongodb"`
Mysql gogorm.Config `yaml:"mysql"`
Postgres gogorm.Config `yaml:"postgres"`
Sqlite gogorm.Config `yaml:"sqlite"`
Clickhouse1 gogorm.Config `yaml:"clickhouse1"`
Redis goredis.Config `yaml:"redis"`
Kafka gokafka.Config `yaml:"kafka"`
Clickhouse goclickhouse.Config `yaml:"clickhouse"`
Es goes.Config `yaml:"es"`
////EsIndex EsIndex `yaml:"es_index"`
Etcd goetcd.Config `yaml:"etcd"`
FeiShu string `yaml:"feishu"`
}
func LoadYamlConfig(yamlFile string, conf interface{}) (err error) {
if yamlFile == "" {
yamlFile = "api-local.yaml"
}
var buf []byte
buf, err = os.ReadFile(yamlFile)
if err != nil {
golog.Error(err.Error())
return
}
if err = yaml.Unmarshal(buf, conf); err != nil {
golog.Error(err.Error())
}
return
}
func LoadJsonConfig(jsonFile string, conf interface{}) (err error) {
if jsonFile == "" {
jsonFile = ".json"
}
var buf []byte
buf, err = os.ReadFile(jsonFile)
if err != nil {
golog.Error(err.Error())
return
}
if err = json.Unmarshal(buf, conf); err != nil {
golog.Error(err.Error())
}
return
}
goio/server-case/controller/common-controller/captcha.go (532 B)
package common_controller
import (
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gif-gif/go.io/goio/server"
"github.com/gin-gonic/gin"
)
type Captcha struct {
request struct {
Width int `json:"width"`
Height int `json:"height"`
}
}
func (this Captcha) DoHandle(ctx *gin.Context) *goserver.Response {
if err := ctx.ShouldBind(&this.request); err != nil {
return goserver.Error(7001, err.Error())
}
rsp := goutils.CaptchaGet(this.request.Width, this.request.Height)
return goserver.SuccessResponse(rsp)
}
goio/server-case/controller/common-controller/file_download.go (1.0 KiB)
package common_controller
import (
gofile "github.com/gif-gif/go.io/go-file"
goserver "github.com/gif-gif/go.io/goio/server"
"github.com/gin-gonic/gin"
)
type FileDownload struct {
}
//func (this FileDownload) DoHandle(ctx *gin.Context) *goserver.Response {
// ds := gofile.NewGoDownload(ctx, "test.csv", ctx.Writer, ctx.Request)
// err := ds.SetFileHeaders()
// file := "/Users/Jerry/Desktop/test/ios_file3.csv"
// err = ds.Output(file)
// if err != nil {
// http.Error(ctx.Writer, "Streaming unsupported!", http.StatusInternalServerError)
// return nil
// }
// return nil
//}
func (this FileDownload) DoHandle(ctx *gin.Context) *goserver.Response {
ds := gofile.NewGoDownload(ctx, "test.csv", ctx.Writer, ctx.Request)
err := ds.SetFileHeaders()
if err != nil {
return nil
}
filePath := "/Users/Jerry/Desktop/test/ios_file3.csv"
err = gofile.ReadLines(filePath, func(chunk string) error {
err = ds.Write([]byte(chunk + "\n"))
return err
})
if err != nil {
ds.Error(err)
return nil
}
return nil
}
goio/server-case/controller/common-controller/health.go (235 B)
package common_controller
import (
"github.com/gif-gif/go.io/goio/server"
"github.com/gin-gonic/gin"
)
type Health struct {
}
func (this Health) DoHandle(ctx *gin.Context) *goserver.Response {
ctx.String(200, "ok")
return nil
}
goio/server-case/controller/common-controller/user_login.go (1.6 KiB)
package common_controller
import (
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gif-gif/go.io/goio/server"
"github.com/gin-gonic/gin"
)
type Login struct {
Request struct {
Account string `json:"account" binding:"required"`
Password string `json:"password" binding:"required"`
CaptchaId string `json:"captcha_id" binding:"required"`
CaptchaCode string `json:"captcha_code" binding:"required"`
}
Response LoginResponse
}
type LoginResponse struct {
Token string `json:"token"`
Username string `json:"username"`
IsSuper bool `json:"is_super"`
}
func (this Login) DoHandle(ctx *gin.Context) *goserver.Response {
if err := ctx.ShouldBind(&this.Request); err != nil {
return goserver.ErrorWithValidate(err, map[string]string{
"account_required": "登录账号 为空",
"password_required": "登录密码 为空",
"captcha_id_required": "验证码 无效",
"captcha_code_required": "验证码 为空",
})
}
if !goutils.CaptchaVerify(this.Request.CaptchaId, this.Request.CaptchaCode) {
return goserver.Error(7001, "验证码错误")
}
var userId int64 = 1
var userName = "tes"
token, err := goserver.CreateToken("AppKey", userId)
if err != nil {
return goserver.Error(7004, "登录失败,提示信息:"+err.Error())
}
this.Response = LoginResponse{
Token: token,
Username: userName,
IsSuper: true,
}
if this.Response.Username == "" {
this.Response.Username = userName
}
goutils.AsyncFunc(func() {
})
ctx.Set("user_id", userId)
ctx.Set("username", userName)
return goserver.SuccessResponse(this.Response)
}
goio/server-case/etc/api-local.yaml (500 B)
env: "dev"
server:
addr: ":10152"
name: "/app/http-web"
app_key: "542a4c9d65fc8f9f"
app_secret: "7abe12ea64d015eb8f3f812fa1ea5508"
prometheus:
host: "0.0.0.0"
port: 10255
# path: "/metrics"
path1: "/metrics"
redis:
addr: "127.0.0.1:6379"
password: ""
db: 0
prefix: "goio:"
etcd:
endpoints:
- "127.0.0.1:23790"
username: ""
password: ""
feishu: "https://open.feishu.cn/open-apis/bot/v2/hook/810d552b-8146-47c3-ae01-13d63007b47d"
goio/server-case/etc/api-prod.yaml (500 B)
env: "dev"
server:
addr: ":10152"
name: "/app/http-web"
app_key: "542a4c9d65fc8f9f"
app_secret: "7abe12ea64d015eb8f3f812fa1ea5508"
prometheus:
host: "0.0.0.0"
port: 10255
# path: "/metrics"
path1: "/metrics"
redis:
addr: "127.0.0.1:6379"
password: ""
db: 0
prefix: "goio:"
etcd:
endpoints:
- "127.0.0.1:23790"
username: ""
password: ""
feishu: "https://open.feishu.cn/open-apis/bot/v2/hook/810d552b-8146-47c3-ae01-13d63007b47d"
goio/server-case/etc/api-test.yaml (500 B)
env: "dev"
server:
addr: ":10152"
name: "/app/http-web"
app_key: "542a4c9d65fc8f9f"
app_secret: "7abe12ea64d015eb8f3f812fa1ea5508"
prometheus:
host: "0.0.0.0"
port: 10255
# path: "/metrics"
path1: "/metrics"
redis:
addr: "127.0.0.1:6379"
password: ""
db: 0
prefix: "goio:"
etcd:
endpoints:
- "127.0.0.1:23790"
username: ""
password: ""
feishu: "https://open.feishu.cn/open-apis/bot/v2/hook/810d552b-8146-47c3-ae01-13d63007b47d"
goio/server-case/main.go (2.3 KiB)
package main
import (
"context"
"flag"
golog "github.com/gif-gif/go.io/go-log"
"github.com/gif-gif/go.io/go-utils/goprometheus/goprometheusx"
"github.com/gif-gif/go.io/goio"
"github.com/gif-gif/go.io/goio/server"
conf "github.com/gif-gif/go.io/goio/server-case/config"
"github.com/gif-gif/go.io/goio/server-case/router"
"github.com/gin-gonic/gin"
"log"
"net/http"
"os"
"os/signal"
"time"
)
var (
yamlFile = flag.String("yaml", "etc/api-local.yaml", "yaml file")
)
func main() {
startSever()
}
func startSever() {
// 初始化命令行参数
goio.FlagInit()
// yaml文件必须
if *yamlFile == "" {
flag.PrintDefaults()
golog.Fatal("yaml file not exist.")
}
// 加载配置文件
confs := &conf.Config{}
err := goserver.LoadYamlConfig(*yamlFile, confs)
if err != nil {
golog.WithTag("main").Error(err)
return
}
goio.Init(confs.Env)
goio.SetupLogDefault()
goio.Setup("")
goprometheusx.Init(confs.Prometheus)
goprometheusx.AlertErr(confs.Server.Name, "main start")
s := goserver.NewServer(
goserver.ServerNameOption("serverName"),
goserver.EnvOption(goio.Env),
//goserver.EnableEncryptionOption("1a3295a2408d553a8458085e7435898e", "119f54388848cb4306f6d2067a4713fce4193504ca368d648196c840ba87da65"),
goserver.PProfEnableOption(false),
goserver.NoLogPathsOption("/captcha/get"),
)
// 路由
router.Routes(s.Group("/"))
// 启动
s.Run(":1000")
}
// 简介的web服务器启动
func simpleServer() {
r := gin.Default()
r.GET("/ping", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{
"message": "pong",
})
})
// 静态页面返回
//r.LoadHTMLGlob("templates/*")
//r.GET("/", func(c *gin.Context) {
// c.HTML(200, "index.tmpl", gin.H{
// "title": "Parse-video",
// })
//})
srv := &http.Server{
Addr: ":1000",
Handler: r,
}
go func() {
// 服务连接
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %s\n", err)
}
}()
// 等待中断信号以优雅地关闭服务器 (设置 5 秒的超时时间)
quit := make(chan os.Signal)
signal.Notify(quit, os.Interrupt)
<-quit
log.Println("Shutdown Server ...")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
log.Fatal("Server Shutdown:", err)
}
log.Println("Server exiting")
}
goio/server-case/router/middleware.go (639 B)
package router
import (
goutils "github.com/gif-gif/go.io/go-utils"
"github.com/gif-gif/go.io/goio/server"
"github.com/gin-gonic/gin"
"strings"
)
func verifySign(c *gin.Context) {
ua := c.GetHeader("User-Agent")
tss := c.GetHeader("X-Request-Timestamp")
sign := c.GetHeader("X-Request-Sign")
if strings.ToLower(sign) != strings.ToLower(goutils.SHA1([]byte(tss+goutils.MD5([]byte(ua))))) {
c.AbortWithStatusJSON(403, goserver.Error(40301, "签名错误"))
return
}
c.Next()
}
func verifyCaptcha(c *gin.Context) {
if 1 == 1 {
c.AbortWithStatusJSON(403, goserver.Error(40301, "验证码错误"))
return
}
c.Next()
}
goio/server-case/router/router.go (534 B)
package router
import (
"github.com/gif-gif/go.io/goio/server"
common_controller2 "github.com/gif-gif/go.io/goio/server-case/controller/common-controller"
"github.com/gin-gonic/gin"
)
func Routes(r *gin.RouterGroup) {
r.POST("/health", goserver.Handler(common_controller2.Health{}))
r.POST("/login", goserver.Handler(common_controller2.Login{}))
r.POST("/captcha/get", goserver.Handler(common_controller2.Captcha{}))
r.POST("/file/download", goserver.Handler(common_controller2.FileDownload{}))
r.Use(verifySign)
{
}
}
goio/status.go (79 B)
package goio
var (
UnknownStatus = 0
StatusValid = 1
StatusInValid = 2
)
goio/test/main.go (524 B)
package main
import (
"fmt"
golog "github.com/gif-gif/go.io/go-log"
goutils "github.com/gif-gif/go.io/go-utils"
)
func main() {
// 加密
key, iv, err := goutils.GenerateAESKeyAndIV()
if err != nil {
golog.WithTag("aes").Error(err)
return
}
fmt.Printf("密钥: %x\n", key)
fmt.Printf("IV: %x\n", iv)
fmt.Println(key, iv)
golog.WithTag("aes1111111").Info(key, iv)
key, err = goutils.GenerateAESKey()
if err != nil {
golog.WithTag("aes").Error(err)
return
}
golog.WithTag("aes22222222").Info(key)
}
main.go (751 B)
package main
import (
"fmt"
golog "github.com/gif-gif/go.io/go-log"
gomessage "github.com/gif-gif/go.io/go-message"
"github.com/gif-gif/go.io/goio"
)
type Config struct {
Name string
FeiShu string
Mode string
}
// 初始化实例
func main() {
c := Config{}
goio.Init(goio.Environment(c.Mode))
//golog.SetAdapter(golog.NewFileAdapter()) //当前工程目录logs/date.log, 可通过这个设置改变日志输出目录
//不设置 golog.SetAdapter 默认控制台输出
golog.WithHook(func(msg *golog.Message) {
if msg.Level > golog.ERROR { //致命错误以上发通知
gomessage.FeiShu(c.FeiShu, fmt.Sprintf(">> %s/%s >> %s",
c.Name, c.Mode, string(msg.JSON())))
}
})
golog.WithTag("goio").ErrorF("Starting tests")
}
🤖 LLM View - CXML Format
Copy the text below and paste it to an LLM for analysis: